Changeset - 57e04d138eb8
[Not reviewed]
Merge feature-cli
0 7 0
Dennis Fink - 9 years ago 2016-03-03 00:16:14
dennis.fink@c3l.lu
Merged dev
7 files changed with 46 insertions and 27 deletions:
0 comments (0 inline, 0 general)
.hgtags
Show inline comments
 
11bc9e2ddf0107cfdc15a78ccb3e4b73fe891184 v5.2.0
 
0c40b177d129979e0436ddb48ac6d80792ae825c v5.2.1
 
1e9bc98a5eec8bf73ebb4019d612828807775cf3 v5.3.0
 
812e073097100155247b2ab698998e604896c37b v5.3.1
 
89496c406287f75bdcb8ffaca83b3014bee3ee95 v5.4.0
 
8d9f6524a1fd22d5e0f9e34be24f93b2d59f15a7 v5.4.1
 
75aa89d1192a128c7c5042c5d365074e2af67a7e v5.4.2
ennstatus/api/model.py
Show inline comments
 
@@ -3,84 +3,87 @@
 
#
 
# This program is free software: you can redistribute it and/or modify
 
# it under the terms of the GNU General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
#
 
# You should have received a copy of the GNU General Public License
 
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 

	
 
import ipaddress
 
import json
 
import functools
 
import statistics
 

	
 
from pathlib import Path
 
from datetime import datetime
 

	
 
import jsonschema
 
import strict_rfc3339
 
import requests
 

	
 
from flask import current_app
 
from pkg_resources import resource_filename
 
from onion_py.manager import Manager
 
from onion_py.caching import OnionSimpleCache
 

	
 
from ..utils import check_ip
 

	
 

	
 
schema = json.load(
 
    open(
 
        resource_filename('ennstatus.api', 'schema/server.json'),
 
        encoding='utf-8'
 
    )
 
)
 

	
 
validate = functools.partial(
 
    jsonschema.validate,
 
    schema=schema,
 
    format_checker=jsonschema.FormatChecker()
 
)
 

	
 
manager = Manager(OnionSimpleCache())
 

	
 

	
 
def calculate_weight(data):
 

	
 
    obj = {}
 

	
 
    for subkey in ('1_week', '1_month', '3_months', '1_year', '5_years'):
 

	
 
        try:
 
            subdata = data[subkey]
 
        except KeyError:
 
            continue
 

	
 
        factor = subdata['factor']
 
        factor = subdata.factor
 

	
 
        values = [x * factor for x in subdata['values'] if x is not None]
 
        values = [x * factor for x in subdata.values if x is not None]
 

	
 
        if values:
 
            obj[subkey] = statistics.mean(values) * 100
 
        else:
 
            obj[subkey] = None
 

	
 
    return obj
 

	
 

	
 
class ServerEncoder(json.JSONEncoder):
 

	
 
    def default(self, obj):
 

	
 
        if isinstance(obj, (ipaddress.IPv4Address, ipaddress.IPv6Address)):
 
            return str(obj)
 

	
 
        if isinstance(obj, datetime):
 
            return strict_rfc3339.timestamp_to_rfc3339_utcoffset(
 
                obj.timestamp()
 
            )
 

	
 
        return json.JSONEncoder.default(self, obj)
 

	
 

	
 
@@ -96,48 +99,49 @@ class ServerDecoder(json.JSONDecoder):
 
                    key, default_obj[key]
 
                )
 
                )
 
                default_obj[key] = ipaddress.ip_address(default_obj[key])
 

	
 
        current_app.logger.debug('Loading last_updated')
 
        default_obj['last_updated'] = datetime.fromtimestamp(
 
            strict_rfc3339.rfc3339_to_timestamp(default_obj['last_updated'])
 
        )
 

	
 
        return default_obj
 

	
 

	
 
class Server:
 

	
 
    def __init__(self, *args, **kwargs):
 

	
 
        self.name = kwargs['name']
 
        self.type = kwargs['type']
 
        self.status = kwargs.get('status')
 
        self.fingerprint = kwargs['fingerprint']
 
        self.last_updated = kwargs['last_updated']
 
        self.country = kwargs['country']
 
        self.bandwidth = kwargs.get('bandwidth')
 
        self.flags = kwargs.get('flags')
 

	
 
        if self.type == 'bridge':
 
            self.obfs = kwargs.get('obfs')
 
            self.fteproxy = kwargs.get('fteproxy')
 
            self.flashproxy = kwargs.get('flashproxy')
 
            self.meek = kwargs.get('meek')
 
        else:
 
            self.ip = kwargs['ip']
 

	
 
            if 'ip6' in kwargs:
 
                self.ip6 = kwargs['ip6']
 

	
 
            default_weights = {
 
                '1_week': None,
 
                '1_month': None,
 
                '3_months': None,
 
                '1_year': None,
 
                '5_years': None
 
            }
 

	
 
            self.mean_consensus_weight = kwargs.get(
 
                'mean_consensus_weight',
 
                default_weights
 
            )
 
@@ -203,58 +207,65 @@ class Server:
 
            raise e
 

	
 
        for key in ('ip', 'ip6'):
 
            if key in server:
 
                address = ipaddress.ip_address(server[key])
 
                if not check_ip(address):
 
                    raise ValueError('{} is not accepted!\n'.format(key))
 
        return True
 

	
 
    def save(self):
 

	
 
        filepath = Path('data') / (self.name.lower() + '.json')
 

	
 
        try:
 
            with filepath.open(mode='w', encoding='utf-8') as f:
 
                json.dump(self.__dict__, f, cls=ServerEncoder)
 
        except Exception as e:
 
            raise e
 

	
 
    def update_weights(self):
 

	
 
        if self.type not in ('exit', 'relay'):
 
            raise NotImplementedError
 

	
 
        url = 'https://onionoo.torproject.org/weights?lookup={}'.format(
 
            self.fingerprint
 
        try:
 
            data = manager.query('weights', lookup=self.fingerprint)
 
        except:
 
            raise NotImplementedError
 

	
 
        if data is not None:
 
            data = data.relays[0]
 

	
 
        self.mean_consensus_weight = calculate_weight(data.consensus_weight)
 
        self.mean_exit_probability = calculate_weight(data.exit_probability)
 
        self.mean_guard_probability = calculate_weight(
 
            data.guard_probability
 
        )
 
        self.mean_middle_probability = calculate_weight(
 
            data.middle_probability
 
        )
 
        self.mean_consensus_weight_fraction = calculate_weight(
 
            data.consensus_weight_fraction
 
        )
 

	
 
        data = requests.get(url)
 
    def update_flags(self):
 

	
 
        try:
 
            data.raise_for_status()
 
        except requests.HTTPError as e:
 
            raise e
 
        else:
 
            data = data.json()['relays'][0]
 
            data = manager.query('details', lookup=self.fingerprint)
 
        except:
 
            raise NotImplementedError
 

	
 
        self.mean_consensus_weight = calculate_weight(data['consensus_weight'])
 
        self.mean_exit_probability = calculate_weight(data['exit_probability'])
 
        self.mean_guard_probability = calculate_weight(
 
            data['guard_probability']
 
        )
 
        self.mean_middle_probability = calculate_weight(
 
            data['middle_probability']
 
        )
 
        self.mean_consensus_weight_fraction = calculate_weight(
 
            data['consensus_weight_fraction']
 
        )
 
        if data is not None:
 
            self.flags = data.relays[0].flags
 
        else:
 
            raise NotImplementedError
 

	
 
    def check_status(self):
 

	
 
        now = datetime.utcnow()
 
        delta = now - self.last_updated
 

	
 
        if delta.seconds >= 3600:
 
            self.status = False
 
        elif delta.seconds >= 600:
 
            self.status = None
ennstatus/api/schema/server.json
Show inline comments
 
@@ -79,48 +79,53 @@
 
                "United States", "United States Minor Outlying Islands",
 
                "Uruguay", "Uzbekistan", "Vanuatu", "Venezuela", "Vietnam",
 
                "Virgin Islands, British", "Virgin Islands, U.S.",
 
                "Wallis and Futuna", "Western Sahara", "Yemen", "Zambia",
 
                "Zimbabwe"
 
            ]
 
        },
 
        "fingerprint": {
 
            "type": "string",
 
            "pattern": "^[a-zA-Z0-9]{40}$"
 
        },
 
        "last_updated": {
 
            "type": "string",
 
            "format": "date-time"
 
        },
 
        "bandwidth": {
 
            "type": [
 
                "string",
 
                "null"
 
            ],
 
            "default": null
 
        },
 
        "type": {
 
            "type": "string"
 
        },
 
        "flags": {
 
            "type": "array",
 
            "items": { "type": "string" },
 
            "uniqueItems": true
 
        }
 
    },
 
    "required": [
 
        "name",
 
        "status",
 
        "last_updated",
 
        "country",
 
        "fingerprint",
 
        "type"
 
    ],
 
    "definitions": {
 
        "weights": {
 
            "type": "object",
 
            "properties": {
 
                "1_week": {
 
                    "$ref": "#/definitions/weights_data"
 
                },
 
                "1_month": {
 
                    "$ref": "#/definitions/weights_data"
 
                },
 
                "3_months": {
 
                    "$ref": "#/definitions/weights_data"
 
                },
 
                "1_year": {
ennstatus/api/views.py
Show inline comments
 
@@ -5,49 +5,48 @@
 
# it under the terms of the GNU General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
#
 
# You should have received a copy of the GNU General Public License
 
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 

	
 
import ipaddress
 
import json
 

	
 
from datetime import datetime
 

	
 
from flask import (Blueprint, request, current_app, jsonify, render_template,
 
                   abort)
 

	
 
from werkzeug.exceptions import BadRequest
 

	
 
import strict_rfc3339
 
import pygeoip
 
import requests
 

	
 
from ennstatus import csrf
 
from ennstatus.status.functions import (single_server, all_servers,
 
                                        all_servers_by_type)
 
from .model import Server
 
from .auth import httpauth
 

	
 
api_page = Blueprint('api', __name__)
 
gi4 = pygeoip.GeoIP('/usr/share/GeoIP/GeoIP.dat', pygeoip.MEMORY_CACHE)
 
gi6 = pygeoip.GeoIP('/usr/share/GeoIP/GeoIPv6.dat', pygeoip.MEMORY_CACHE)
 

	
 

	
 
@csrf.exempt
 
@api_page.route('/update', methods=('POST',))
 
@httpauth.login_required
 
def update():
 

	
 
    current_app.logger.info('Handling update')
 

	
 
    try:
 
        servers = current_app.config['ENNSTATUS_SERVERS']
 
    except KeyError as e:
 
        current_app.logger.error(str(e))
 
        return abort(500)
 
@@ -99,50 +98,52 @@ def update():
 
                data['ip'] = ip
 
            elif temp_ip.verison == 6:
 
                data['ip6'] = ip
 

	
 
    try:
 
        country = gi4.country_name_by_addr(ip)
 
    except pygeoip.GeoIPError:
 
        country = gi6.country_name_by_addr(ip)
 

	
 
    data['country'] = country
 
    data['last_updated'] = strict_rfc3339.timestamp_to_rfc3339_utcoffset(
 
        datetime.utcnow().timestamp()
 
    )
 

	
 
    try:
 
        server = Server.from_dict(data)
 
    except Exception as e:
 
        current_app.logger.warning(' '.join([str(e), str(data)]))
 
        return str(e), 409, {'Content-Type': 'text/plain'}
 

	
 
    try:
 
        server.update_weights()
 
    except NotImplementedError:
 
        pass
 
    except requests.HTTPError as e:
 
        current_app.logger.error(str(e), exc_info=True)
 

	
 
    try:
 
        server.update_flags()
 
    except NotImplementedError:
 
        pass
 

	
 
    try:
 
        server.save()
 
    except Exception as e:
 
        current_app.logger.error(str(e))
 
        return str(e), 500, {'Content-Type': 'text/plain'}
 

	
 
    current_app.logger.info('Return result')
 
    return (
 
        server.json(), 201,
 
        {
 
            'Location': '/api/export/json/single?server_name={}'.format(
 
                server.name
 
            )
 
        }
 
    )
 

	
 

	
 
@api_page.route('/export', defaults={'server_type': 'all'})
 
@api_page.route(('/export/<any("all", "exit", "bridge", "relay", "single")'
 
                 ':server_type>'))
 
def export(server_type):
 

	
requirements.in
Show inline comments
 
Babel
 
click
 
Flask-Bootstrap
 
Flask-HTTPAuth
 
Flask-Mail
 
Flask-Moment
 
Flask-WTF
 
Flask
 
jsonschema
 
pygeoip
 
python-gnupg
 
requests
 
strict-rfc3339
 
OnionPy
requirements.txt
Show inline comments
 
#
 
# This file is autogenerated by pip-compile
 
# Make changes in requirements.in, then run this to update:
 
#
 
#    pip-compile requirements.in
 
#
 
Babel==2.2.0
 
blinker==1.4              # via flask-mail
 
click==6.3
 
dominate==2.1.17          # via flask-bootstrap
 
Flask-Bootstrap==3.3.5.7
 
Flask-HTTPAuth==2.7.2
 
Flask-Mail==0.9.1
 
Flask-Moment==0.5.1
 
Flask-WTF==0.12
 
Flask==0.10.1             # via flask-bootstrap, flask-httpauth, flask-mail, flask-moment, flask-wtf
 
itsdangerous==0.24        # via flask
 
Jinja2==2.8               # via flask
 
jsonschema==2.5.1
 
MarkupSafe==0.23          # via jinja2
 
onionpy==0.3.2
 
pygeoip==0.3.2
 
python-gnupg==0.3.8
 
pytz==2015.7              # via babel
 
requests==2.9.1
 
strict-rfc3339==0.6
 
visitor==0.1.2            # via flask-bootstrap
 
Werkzeug==0.11.4          # via flask, flask-wtf
 
WTForms==2.1              # via flask-wtf
setup.py
Show inline comments
 
from setuptools import setup, find_packages
 

	
 

	
 
def _get_requirements():
 

	
 
    with open('requirements.in', encoding='utf-8') as f:
 
        lines = f.readlines()
 

	
 
    lines = [line[:-1] for line in lines if not line.startswith('#')]
 
    return lines
 

	
 

	
 
setup(name='Ennstatus',
 
      version='5.4.1-dev',
 
      version='5.4.2-dev',
 
      description=('Ennstatus provides the user with vital information about '
 
                   'the status of the organizations Tor servers.'),
 
      author='Frënn vun der Ënn',
 
      author_email='info@enn.lu',
 
      url='https://bitbucket.org/fvde/ennstatus',
 
      license='GPLv3+',
 
      packages=find_packages(),
 
      install_requires=_get_requirements(),
 
      include_package_data=True,
 
      entry_points={
 
          'console_scripts': [
 
              'ennstatuscli = ennstatus.cli:cli',
 
          ]
 
      },
 
      classifiers=['Development Status :: 5 - Production/Stable',
 
                   'Environment :: Web Environment',
 
                   'Operating System :: POSIX',
 
                   'Programming Language :: Python',
 
                   'Programming Language :: Python :: 3',
 
                   'Programming Language :: Python :: 3.3',
 
                   'Topic :: Internet',
 
                   'Topic :: Internet :: WWW/HTTP',
 
                   'Topic :: Internet :: WWW/HTTP :: Dynamic Content',
 
                   'Topic :: Internet :: WWW/HTTP :: WSGI',
0 comments (0 inline, 0 general)