# HG changeset patch # User Dennis Fink # Date 2015-08-21 16:16:40 # Node ID 1585155d7c8ae46773368b64150ed94a5e473d98 # Parent e701f9a8e6063cc1f62da335cbf36d3c876075ab Use new model for servers diff --git a/ennstatus/api/functions.py b/ennstatus/api/functions.py deleted file mode 100644 --- a/ennstatus/api/functions.py +++ /dev/null @@ -1,155 +0,0 @@ - -import re -import json -import ipaddress - -from datetime import datetime - -import pygeoip - -from flask import current_app - -from ennstatus.status.functions import _send_mail - - -FINGERPRINT_REGEX = re.compile(r'^[A-Z0-9]{40}$', re.I) - -DATE_FORMAT = '%d-%m-%Y %H:%M:%S' - -gi4 = pygeoip.GeoIP('/usr/share/GeoIP/GeoIP.dat', pygeoip.MEMORY_CACHE) -gi6 = pygeoip.GeoIP('/usr/share/GeoIP/GeoIPv6.dat', pygeoip.MEMORY_CACHE) - -mail_cache = dict() - - -def check_bridge(key, server): - - if key not in server: - raise ValueError('%s key not present!\n' % key) - else: - if not isinstance(server[key], bool): - error_message = ('%s has not the right type!' - ' Needs to be a boolean!\n') % key - raise ValueError(error_message) - - -def check_json_format(server): - - for key in ('server_type', 'server_name', 'tor_status', 'fingerprint'): - if key not in server: - raise ValueError('%s key not present!\n' % key) - - if server['server_type'] not in ('Exit', 'Relay', 'Bridge'): - error_message = ('server_type has not the right content!' - ' is: %s must be one of: Exit, Relay or Bridge\n') \ - % server['server_type'] - raise ValueError(error_message) - - if not server['tor_status'] in ('Online', 'Offline'): - error_message = ('tor_status has not the right content!' - ' is: %s must be one of: Online or Offline\n') \ - % server['tor_status'] - - if FINGERPRINT_REGEX.match(server['fingerprint']) is None: - raise ValueError('fingerprint has not the right format!\n') - - if server['server_type'] == 'Bridge': - for key in ('obfs', 'fteproxy', 'flashproxy', 'meek'): - check_bridge(key, server) - - if 'ip' in server: - try: - address = ipaddress.IPv4Address(server['ip']) - - if any([address.is_private, address.is_multicast, - address.is_unspecified, address.is_reserved, - address.is_loopback]): - raise ValueError('ip is not accepted!\n') - - except ipaddress.AddressValueError: - raise ValueError('ip is not the right format!\n') - - if 'ip6' in server: - try: - address = ipaddress.IPv6Address(server['ip6']) - if any([address.is_private, address.is_multicast, - address.is_unspecified, address.is_reserved, - address.is_loopback]): - raise ValueError('ip6 is not accepted!\n') - - except ipaddress.AddressValueError: - raise ValueError('ip6 is not the right format!\n') - - return True - - -def _send_offline_mail(server_name, last_updated): - - current_app.logger.info('Sending tor status offline mail!') - subject = '[Ennstatus] %s Tor status went offline' - _send_mail(server_name, last_updated, subject) - mail_cache[server_name] = datetime.utcnow() - - -def update_server(server, ip): - - server['last_updated'] = datetime.utcnow().strftime(DATE_FORMAT) - server['server_status'] = 'Online' - - if ip.version == 4: - server['country'] = gi4.country_name_by_addr(str(ip)) - elif ip.version == 6: - server['country'] = gi6.country_name_by_addr(str(ip)) - - server_name = server['server_name'] - - if server['server_type'] == 'Bridge': - if 'ip' in server: - del server['ip'] - - if 'ip6' in server: - del server['ip6'] - else: - for key in ('obfs', 'fteproxy', 'flashproxy', 'meek'): - if key in server: - del server[key] - - if ip.version == 4: - if 'ip' not in server: - server['ip'] = str(ip) - elif ip.version == 6: - if 'ip6' not in server: - server['ip6'] = str(ip) - - try: - filename = ''.join(['data/', server_name.lower(), '.json']) - - with open(filename, 'w', encoding='utf-8') as fb: - json.dump(server, fb) - except Exception as e: - return e - - if server['tor_status'] == 'Offline': - - if server_name in mail_cache: - - current_app.logger.debug('Mail cache is %s' % str(mail_cache)) - - send_date = mail_cache[server_name] - now = datetime.utcnow() - delta = now - send_date - - if delta.seconds <= 7200: - current_app.logger.debug('Server is in mail cache but not old enough to resend mail') - return server - else: - current_app.logger.debug('Server tor status is offline for more than 7200 seconds') - _send_offline_mail(server_name, server['last_updated']) - else: - current_app.logger.debug('Server is not in mail cache') - _send_offline_mail(server_name, server['last_updated']) - elif server_name in mail_cache: - current_app.logger.debug('Removing server from mail cache') - del mail_cache[server_name] - - return server diff --git a/ennstatus/api/model.py b/ennstatus/api/model.py new file mode 100644 --- /dev/null +++ b/ennstatus/api/model.py @@ -0,0 +1,241 @@ +import ipaddress +import json +import functools +import statistics + +from pathlib import Path +from datetime import datetime + +import jsonschema +import strict_rfc3339 +import requests + +from flask import current_app +from pkg_resources import resource_filename + +schema = json.load( + open( + resource_filename('ennstatus.api', 'schema/server.json'), + encoding='utf-8' + ) +) + +validate = functools.partial( + jsonschema.validate, + schema=schema, + format_checker=jsonschema.FormatChecker() +) + + +def calculate_weight(data): + + obj = {} + + for subkey in ('1_week', '1_month', '3_months', '1_year', '5_years'): + + subdata = data[subkey] + factor = subdata['factor'] + + values = [x * factor for x in subdata['values'] if x is not None] + + if values: + obj[subkey] = statistics.mean(values) * 100 + else: + obj[subkey] = None + + return obj + + +class ServerEncoder(json.JSONEncoder): + + def default(self, obj): + + if isinstance(obj, (ipaddress.IPv4Address, ipaddress.IPv6Address)): + return str(obj) + + if isinstance(obj, datetime): + return strict_rfc3339.timestamp_to_rfc3339_utcoffset( + obj.timestamp() + ) + + return json.JSONEncoder.default(self, obj) + + +class ServerDecoder(json.JSONDecoder): + + def decode(self, json_string): + + default_obj = super().decode(json_string) + + for key in ('ip', 'ip6'): + if key in default_obj: + current_app.logger.debug('{}: {}'.format(key, default_obj[key])) + default_obj[key] = ipaddress.ip_address(default_obj[key]) + + current_app.logger.debug('Loading last_updated') + default_obj['last_updated'] = datetime.fromtimestamp( + strict_rfc3339.rfc3339_to_timestamp(default_obj['last_updated']) + ) + + return default_obj + + +class Server: + + def __init__(self, *args, **kwargs): + + self.name = kwargs['name'] + self.type = kwargs['type'] + self.status = kwargs.get('status') + self.tor_status = kwargs.get('tor_status') + self.fingerprint = kwargs['fingerprint'] + self.last_updated = kwargs['last_updated'] + self.country = kwargs['country'] + self.bandwith = kwargs.get('bandwith') + + if self.type == 'bridge': + self.obfs = kwargs.get('obfs') + self.fteproxy = kwargs.get('fteproxy') + self.flashproxy = kwargs.get('flashproxy') + self.meek = kwargs.get('meek') + else: + self.ip = kwargs['ip'] + + if 'ip6' in kwargs: + self.ip6 = kwargs['ip6'] + + default_weights = { + '1_week': None, + '1_month': None, + '3_months': None, + '1_year': None, + '5_years': None + } + + self.mean_consensus_weight = kwargs.get( + 'mean_consensus_weight', + default_weights + ) + self.mean_guard_probability = kwargs.get( + 'mean_guard_probability', + default_weights + ) + self.mean_exit_probability = kwargs.get( + 'mean_exit_probability', + default_weights + ) + self.mean_consensus_weight_fraction = kwargs.get( + 'mean_consensus_weight_fraction', + default_weights + ) + self.mean_middle_probability = kwargs.get( + 'mean_middle_probability', + default_weights + ) + + @classmethod + def from_file_by_name(cls, name): + + filepath = Path('data') / (name.lower() + '.json') + + current_app.logger.info('Loading {}'.format(str(filepath))) + if filepath.exists() and filepath.is_file(): + try: + with filepath.open(encoding='utf-8') as f: + data = json.load(f, cls=ServerDecoder) + except (IOError, ValueError): + current_app.logger.error('IOError or ValueError') + return False + else: + return cls(**data) + else: + current_app.logger.error('File error!') + return False + + @classmethod + def from_json(cls, server): + + try: + if cls.check_json_format(json.loads(server)): + decoded = json.loads(server, cls=ServerDecoder) + return cls(**decoded) + except (jsonschema.ValidationError, ValueError) as e: + raise e + + @classmethod + def from_dict(cls, server): + return cls.from_json(json.dumps(server)) + + def json(self): + return json.dumps(self.__dict__, cls=ServerEncoder) + + @staticmethod + def check_json_format(server): + + try: + validate(server) + except jsonschema.ValidationError as e: + raise e + + for key in ('ip', 'ip6'): + if key in server: + address = ipaddress.ip_address(server[key]) + + if any({address.is_private, address.is_multicast, + address.is_unspecified, address.is_reserved, + address.is_loopback}): + raise ValueError('{} is not accepted!\n'.format(key)) + + return True + + def save(self): + + filepath = Path('data') / (self.name.lower() + '.json') + + try: + with filepath.open(mode='w', encoding='utf-8') as f: + json.dump(self.__dict__, f, cls=ServerEncoder) + except Exception as e: + raise e + + def update_weights(self): + + if self.type not in ('exit', 'relay'): + raise NotImplementedError + + url = 'https://onionoo.torproject.org/weights?lookup={}'.format( + self.fingerprint + ) + + data = requests.get(url) + + try: + data.raise_for_status() + except requests.HTTPError as e: + raise e + else: + data = data.json()['relays'][0] + + self.mean_consensus_weight = calculate_weight(data['consensus_weight']) + self.mean_exit_probability = calculate_weight(data['exit_probability']) + self.mean_guard_probability = calculate_weight( + data['guard_probability'] + ) + self.mean_middle_probability = calculate_weight( + data['middle_probability'] + ) + self.mean_consensus_weight_fraction = calculate_weight( + data['consensus_weight_fraction'] + ) + + def check_status(self): + + now = datetime.utcnow() + delta = now - self.last_updated + + if delta.seconds >= 3600: + self.status = False + self.tor_status = False + return False + else: + return True diff --git a/ennstatus/api/views.py b/ennstatus/api/views.py --- a/ennstatus/api/views.py +++ b/ennstatus/api/views.py @@ -1,13 +1,22 @@ import ipaddress +import json + +from datetime import datetime from flask import (Blueprint, request, current_app, jsonify, render_template, abort) -from ennstatus.api.functions import check_json_format, update_server +import strict_rfc3339 +import pygeoip + from ennstatus.status.functions import (single_server, all_servers, all_servers_by_type) +from .model import Server + api_page = Blueprint('api', __name__) +gi4 = pygeoip.GeoIP('/usr/share/GeoIP/GeoIP.dat', pygeoip.MEMORY_CACHE) +gi6 = pygeoip.GeoIP('/usr/share/GeoIP/GeoIPv6.dat', pygeoip.MEMORY_CACHE) @api_page.route('/update', methods=('POST',)) @@ -19,21 +28,21 @@ def update(): else: accepted_ips = current_app.config.get('ENNSTATUS_ACCEPTED_IPS', []) - json = request.get_json() - if json is None: + if request.remote_addr not in accepted_ips: + current_app.logger.warn('Unallowed IP %s tried to update data!' + % request.remote_addr) + return 'IP not allowed!\n', 403, {'Content-Type': 'text/plain'} + + data = request.get_json() + + if data is None: current_app.logger.info('No JSON data supplied!') return 'No JSON data supplied!\n', 400, {'Content-Type': 'text/plain'} - try: - check_json_format(json) - except ValueError as e: - current_app.logger.warning(' '.join([str(e), str(json)])) - return str(e), 409, {'Content-Type': 'text/plain'} - - if 'ip' in json: - ip = json['ip'] - elif 'ip6' in json: - ip = json['ip6'] + if 'ip' in data: + ip = data['ip'] + elif 'ip6' in data: + ip = data['ip6'] else: ip = request.remote_addr @@ -42,25 +51,37 @@ def update(): except ipaddress.AddressValueError: return 'IP not allowed!\n', 403, {'Content-Type': 'text/plain'} - if request.remote_addr not in accepted_ips: - current_app.logger.warn('Unallowed IP %s tried to update data!' - % ip) - return 'IP not allowed!\n', 403, {'Content-Type': 'text/plain'} + if ip.version == 4: + data['country'] = gi4.country_name_by_addr(str(ip)) + elif ip.version == 6: + data['country'] = gi6.country_name_by_addr(str(ip)) + else: + data['country'] = None - current_app.logger.info(str(json)) - server = update_server(server=json, ip=ip) + data['last_updated'] = strict_rfc3339.timestamp_to_rfc3339_utcoffset( + datetime.utcnow().timestamp() + ) + data['status'] = True - if server: - current_app.logger.info('Return result') - current_app.logger.info(str(server)) - return (jsonify(server), 201, - {'Location': '/api/export/json/single?server_name=%s' - % server['server_name']}) + try: + server = Server.from_json(json.dumps(data)) + except Exception as e: + current_app.logger.warning(' '.join([str(e), str(data)])) + return str(e), 409, {'Content-Type': 'text/plain'} + + if server.type in ('exit', 'relay'): + server.update_weights() - else: - current_app.logger.error('Unexpected error: %s' % server, - exc_info=True) - return abort(500) + try: + server.save() + except Exception as e: + current_app.logger.error(str(e)) + return str(e), 500, {'Content-Type': 'text/plain'} + + current_app.logger.info('Return result') + return (server.json(), 201, + {'Location': '/api/export/json/single?server_name=%s' + % server.name}) @api_page.route('/export', defaults={'server_type': 'all', @@ -80,7 +101,8 @@ def export(export_format, server_type): if server: if export_format == 'json': current_app.logger.info('Returning server as json!') - return jsonify(server) + return (server.json(), 200, + {'Content-Type': 'application/json'}) else: current_app.logger.info('Returning server as xml!') return ( @@ -100,15 +122,14 @@ def export(export_format, server_type): else: if server_type == 'all': current_app.logger.info('Getting all servers!') - servers = list(all_servers()) + servers = [server.json() for server in all_servers()] else: current_app.logger.info('Getting all %s!' % server_type) - servers = list(all_servers_by_type(server_type.capitalize())) + servers = list(all_servers_by_type(server_type.lower())) if export_format == 'json': - response = {'enn_network': servers} current_app.logger.info('Returning as json!') - return jsonify(response) + return str(servers) else: current_app.logger.info('Returning as xml!') return (render_template('api/export/xml/network.xml', diff --git a/ennstatus/status/functions.py b/ennstatus/status/functions.py --- a/ennstatus/status/functions.py +++ b/ennstatus/status/functions.py @@ -8,6 +8,7 @@ from datetime import datetime from flask import current_app from flask_mail import Mail, Message +from ..api.model import Server mail = Mail() @@ -99,29 +100,33 @@ def _load_single_server(filename): def single_server(name): - filename = ''.join(['data/', name, '.json']) - return _load_single_server(filename) + server = Server.from_file_by_name(name) + + if server: + server.check_status() + + return server def _get_json_files(root, files): for f in files: if f.endswith('.json'): - yield os.path.join(root, f) + yield f def all_servers(): for root, _, files in os.walk('data'): for f in _get_json_files(root, files): - yield _load_single_server(f) + yield single_server(f[:-5]) def all_servers_by_type(type): for server in all_servers(): try: - if server['server_type'] == type: + if server.type == type: yield server except TypeError: continue @@ -133,7 +138,7 @@ def split_all_servers_to_types(): for server in all_servers(): try: - servers[server['server_type']].append(server) + servers[server.type].append(server) except TypeError: continue diff --git a/ennstatus/status/views.py b/ennstatus/status/views.py --- a/ennstatus/status/views.py +++ b/ennstatus/status/views.py @@ -13,8 +13,8 @@ def index(): servers = split_all_servers_to_types() current_app.logger.info('Returning servers') - return render_template('status/index.html', exit=servers['Exit'], - relay=servers['Relay'], bridge=servers['Bridge']) + return render_template('status/index.html', exit=servers['exit'], + relay=servers['relay'], bridge=servers['bridge']) @status_page.route('/exit') diff --git a/ennstatus/templates/api/export/xml/server.xml b/ennstatus/templates/api/export/xml/server.xml --- a/ennstatus/templates/api/export/xml/server.xml +++ b/ennstatus/templates/api/export/xml/server.xml @@ -1,20 +1,20 @@ - {{ server['server_type'] }} - {{ server['server_name'] }} - {{ server['server_status'] }} - {{ server['tor_status'] }} - {{ server['country'] }} - {{ server['last_updated'] }} - {{ server['fingerprint'] }} - {% if server['server_type'] == 'bridge' %} - {{ server['obfs'] }} - {{ server['fteproxy'] }} - {{ server['flashproxy'] }} - {{ server['meek'] }} + {{ server.type }} + {{ server.name }} + {{ server.status }} + {{ server.tor_status }} + {{ server.country }} + {{ server.last_updated }} + {{ serverfingerprint }} + {% if server.type == 'bridge' %} + {{ server.obfs }} + {{ server.fteproxy }} + {{ server.flashproxy }} + {{ server.meek }} {% else %} - {{ server['ip'] }} + {{ server.ip }} {% if 'ip6' in server %} - {{ server['ip6'] + {{ server.ip6 {% endif %} {% endif %} diff --git a/ennstatus/templates/status/index.html b/ennstatus/templates/status/index.html --- a/ennstatus/templates/status/index.html +++ b/ennstatus/templates/status/index.html @@ -14,13 +14,13 @@

No servers found!

{% else %} {% if exit %} - {{ macros.create_server_table('Exit', exit) }} + {{ macros.create_server_table('exit', exit) }} {% endif %} {% if relay %} - {{ macros.create_server_table('Relay', relay) }} + {{ macros.create_server_table('relay', relay) }} {% endif %} {% if bridge %} - {{ macros.create_server_table('Bridge', bridge) }} + {{ macros.create_server_table('bridge', bridge) }} {% endif %} {% endif %} diff --git a/ennstatus/templates/status/macros.html b/ennstatus/templates/status/macros.html --- a/ennstatus/templates/status/macros.html +++ b/ennstatus/templates/status/macros.html @@ -1,7 +1,7 @@ {% macro colorize_status(status) %} - {% if status == "Online" %} + {% if status %} {% set color = "text-success" %} - {% elif status == "Unknown" %} + {% elif status is none %} {% set color = "text-warning" %} {% else %} {% set color = "text-danger" %} @@ -24,7 +24,7 @@ {% endmacro %} {% macro create_fingerprint(fingerprint, server_type) %} - {% if server_type in ('Exit', 'Relay') %} + {% if server_type in ('exit', 'relay') %} {% set url_type = 'relay' %} {% else %} {% set url_type = 'bridge' %} @@ -49,7 +49,7 @@ {% else %} {% set headers = ('#', 'Name', 'Server Status', 'Tor Status', 'Country', 'Fingerprint', 'OBFS', 'FTEProxy', 'Flashproxy', 'meek', 'Last Updated (UTC)') %} {% endif %} -

{{ server_type }}

+

{{ server_type.capitalize() }}

@@ -59,37 +59,37 @@ - {% for server in servers|sort(attribute='server_name')|sort(attribute='country') %} - + {% for server in servers|sort(attribute='name')|sort(attribute='country') %} + - {% if server_type in ('Exit', 'Relay') %} - - - + {% if server_type in ('exit', 'relay') %} + + + {% else %} - {% if server['server_name'] in config['ENNSTATUS_BRIDGE_PROGRAM'] %} - + {% if server.name in config['ENNSTATUS_BRIDGE_PROGRAM'] %} + {% else %} - + {% endif %} {% endif %} - {% for status in (server['server_status'], server['tor_status']) %} + {% for status in (server.status, server.tor_status) %} {% endfor %} - - - {% if server_type == 'Bridge' %} - - - - + + + {% if server_type == 'bridge' %} + + + + {% endif %} {% endfor %}
{{ loop.index }}{{ create_name(server['server_name']) }}{{ server['ip'] }}{{ server.get('ip6', 'N/A') }}{{ create_name(server.name) }}{{ server.ip }}{{ server.ip6 or 'N/A' }}{{ server['server_name'] }}{{ server.name }}{{ server['server_name'] }}{{ server.name }}{{ colorize_status(status) }}{{ create_country(server['country']) }}{{ create_fingerprint(server['fingerprint'], server['server_type']) }}{{ colorize_obfs(server['obfs']) }}{{ colorize_obfs(server['fteproxy']) }}{{ colorize_obfs(server['flashproxy']) }}{{ colorize_obfs(server['meek']) }}{{ create_country(server.country) }}{{ create_fingerprint(server.fingerprint, server.type) }}{{ colorize_obfs(server.obfs) }}{{ colorize_obfs(server.fteproxy) }}{{ colorize_obfs(server.flashproxy) }}{{ colorize_obfs(server.meek) }}{{ server['last_updated'] }}
- {% if server_type == 'Bridge' %} + {% if server_type == 'bridge' %}