diff --git a/ennstatus/api/functions.py b/ennstatus/api/functions.py
deleted file mode 100644
--- a/ennstatus/api/functions.py
+++ /dev/null
@@ -1,155 +0,0 @@
-
-import re
-import json
-import ipaddress
-
-from datetime import datetime
-
-import pygeoip
-
-from flask import current_app
-
-from ennstatus.status.functions import _send_mail
-
-
-FINGERPRINT_REGEX = re.compile(r'^[A-Z0-9]{40}$', re.I)
-
-DATE_FORMAT = '%d-%m-%Y %H:%M:%S'
-
-gi4 = pygeoip.GeoIP('/usr/share/GeoIP/GeoIP.dat', pygeoip.MEMORY_CACHE)
-gi6 = pygeoip.GeoIP('/usr/share/GeoIP/GeoIPv6.dat', pygeoip.MEMORY_CACHE)
-
-mail_cache = dict()
-
-
-def check_bridge(key, server):
-
- if key not in server:
- raise ValueError('%s key not present!\n' % key)
- else:
- if not isinstance(server[key], bool):
- error_message = ('%s has not the right type!'
- ' Needs to be a boolean!\n') % key
- raise ValueError(error_message)
-
-
-def check_json_format(server):
-
- for key in ('server_type', 'server_name', 'tor_status', 'fingerprint'):
- if key not in server:
- raise ValueError('%s key not present!\n' % key)
-
- if server['server_type'] not in ('Exit', 'Relay', 'Bridge'):
- error_message = ('server_type has not the right content!'
- ' is: %s must be one of: Exit, Relay or Bridge\n') \
- % server['server_type']
- raise ValueError(error_message)
-
- if not server['tor_status'] in ('Online', 'Offline'):
- error_message = ('tor_status has not the right content!'
- ' is: %s must be one of: Online or Offline\n') \
- % server['tor_status']
-
- if FINGERPRINT_REGEX.match(server['fingerprint']) is None:
- raise ValueError('fingerprint has not the right format!\n')
-
- if server['server_type'] == 'Bridge':
- for key in ('obfs', 'fteproxy', 'flashproxy', 'meek'):
- check_bridge(key, server)
-
- if 'ip' in server:
- try:
- address = ipaddress.IPv4Address(server['ip'])
-
- if any([address.is_private, address.is_multicast,
- address.is_unspecified, address.is_reserved,
- address.is_loopback]):
- raise ValueError('ip is not accepted!\n')
-
- except ipaddress.AddressValueError:
- raise ValueError('ip is not the right format!\n')
-
- if 'ip6' in server:
- try:
- address = ipaddress.IPv6Address(server['ip6'])
- if any([address.is_private, address.is_multicast,
- address.is_unspecified, address.is_reserved,
- address.is_loopback]):
- raise ValueError('ip6 is not accepted!\n')
-
- except ipaddress.AddressValueError:
- raise ValueError('ip6 is not the right format!\n')
-
- return True
-
-
-def _send_offline_mail(server_name, last_updated):
-
- current_app.logger.info('Sending tor status offline mail!')
- subject = '[Ennstatus] %s Tor status went offline'
- _send_mail(server_name, last_updated, subject)
- mail_cache[server_name] = datetime.utcnow()
-
-
-def update_server(server, ip):
-
- server['last_updated'] = datetime.utcnow().strftime(DATE_FORMAT)
- server['server_status'] = 'Online'
-
- if ip.version == 4:
- server['country'] = gi4.country_name_by_addr(str(ip))
- elif ip.version == 6:
- server['country'] = gi6.country_name_by_addr(str(ip))
-
- server_name = server['server_name']
-
- if server['server_type'] == 'Bridge':
- if 'ip' in server:
- del server['ip']
-
- if 'ip6' in server:
- del server['ip6']
- else:
- for key in ('obfs', 'fteproxy', 'flashproxy', 'meek'):
- if key in server:
- del server[key]
-
- if ip.version == 4:
- if 'ip' not in server:
- server['ip'] = str(ip)
- elif ip.version == 6:
- if 'ip6' not in server:
- server['ip6'] = str(ip)
-
- try:
- filename = ''.join(['data/', server_name.lower(), '.json'])
-
- with open(filename, 'w', encoding='utf-8') as fb:
- json.dump(server, fb)
- except Exception as e:
- return e
-
- if server['tor_status'] == 'Offline':
-
- if server_name in mail_cache:
-
- current_app.logger.debug('Mail cache is %s' % str(mail_cache))
-
- send_date = mail_cache[server_name]
- now = datetime.utcnow()
- delta = now - send_date
-
- if delta.seconds <= 7200:
- current_app.logger.debug('Server is in mail cache but not old enough to resend mail')
- return server
- else:
- current_app.logger.debug('Server tor status is offline for more than 7200 seconds')
- _send_offline_mail(server_name, server['last_updated'])
- else:
- current_app.logger.debug('Server is not in mail cache')
- _send_offline_mail(server_name, server['last_updated'])
- elif server_name in mail_cache:
- current_app.logger.debug('Removing server from mail cache')
- del mail_cache[server_name]
-
- return server
diff --git a/ennstatus/api/model.py b/ennstatus/api/model.py
new file mode 100644
--- /dev/null
+++ b/ennstatus/api/model.py
@@ -0,0 +1,241 @@
+import ipaddress
+import json
+import functools
+import statistics
+
+from pathlib import Path
+from datetime import datetime
+
+import jsonschema
+import strict_rfc3339
+import requests
+
+from flask import current_app
+from pkg_resources import resource_filename
+
+schema = json.load(
+ open(
+ resource_filename('ennstatus.api', 'schema/server.json'),
+ encoding='utf-8'
+ )
+)
+
+validate = functools.partial(
+ jsonschema.validate,
+ schema=schema,
+ format_checker=jsonschema.FormatChecker()
+)
+
+
+def calculate_weight(data):
+
+ obj = {}
+
+ for subkey in ('1_week', '1_month', '3_months', '1_year', '5_years'):
+
+ subdata = data[subkey]
+ factor = subdata['factor']
+
+ values = [x * factor for x in subdata['values'] if x is not None]
+
+ if values:
+ obj[subkey] = statistics.mean(values) * 100
+ else:
+ obj[subkey] = None
+
+ return obj
+
+
+class ServerEncoder(json.JSONEncoder):
+
+ def default(self, obj):
+
+ if isinstance(obj, (ipaddress.IPv4Address, ipaddress.IPv6Address)):
+ return str(obj)
+
+ if isinstance(obj, datetime):
+ return strict_rfc3339.timestamp_to_rfc3339_utcoffset(
+ obj.timestamp()
+ )
+
+ return json.JSONEncoder.default(self, obj)
+
+
+class ServerDecoder(json.JSONDecoder):
+
+ def decode(self, json_string):
+
+ default_obj = super().decode(json_string)
+
+ for key in ('ip', 'ip6'):
+ if key in default_obj:
+ current_app.logger.debug('{}: {}'.format(key, default_obj[key]))
+ default_obj[key] = ipaddress.ip_address(default_obj[key])
+
+ current_app.logger.debug('Loading last_updated')
+ default_obj['last_updated'] = datetime.fromtimestamp(
+ strict_rfc3339.rfc3339_to_timestamp(default_obj['last_updated'])
+ )
+
+ return default_obj
+
+
+class Server:
+
+ def __init__(self, *args, **kwargs):
+
+ self.name = kwargs['name']
+ self.type = kwargs['type']
+ self.status = kwargs.get('status')
+ self.tor_status = kwargs.get('tor_status')
+ self.fingerprint = kwargs['fingerprint']
+ self.last_updated = kwargs['last_updated']
+ self.country = kwargs['country']
+ self.bandwith = kwargs.get('bandwith')
+
+ if self.type == 'bridge':
+ self.obfs = kwargs.get('obfs')
+ self.fteproxy = kwargs.get('fteproxy')
+ self.flashproxy = kwargs.get('flashproxy')
+ self.meek = kwargs.get('meek')
+ else:
+ self.ip = kwargs['ip']
+
+ if 'ip6' in kwargs:
+ self.ip6 = kwargs['ip6']
+
+ default_weights = {
+ '1_week': None,
+ '1_month': None,
+ '3_months': None,
+ '1_year': None,
+ '5_years': None
+ }
+
+ self.mean_consensus_weight = kwargs.get(
+ 'mean_consensus_weight',
+ default_weights
+ )
+ self.mean_guard_probability = kwargs.get(
+ 'mean_guard_probability',
+ default_weights
+ )
+ self.mean_exit_probability = kwargs.get(
+ 'mean_exit_probability',
+ default_weights
+ )
+ self.mean_consensus_weight_fraction = kwargs.get(
+ 'mean_consensus_weight_fraction',
+ default_weights
+ )
+ self.mean_middle_probability = kwargs.get(
+ 'mean_middle_probability',
+ default_weights
+ )
+
+ @classmethod
+ def from_file_by_name(cls, name):
+
+ filepath = Path('data') / (name.lower() + '.json')
+
+ current_app.logger.info('Loading {}'.format(str(filepath)))
+ if filepath.exists() and filepath.is_file():
+ try:
+ with filepath.open(encoding='utf-8') as f:
+ data = json.load(f, cls=ServerDecoder)
+ except (IOError, ValueError):
+ current_app.logger.error('IOError or ValueError')
+ return False
+ else:
+ return cls(**data)
+ else:
+ current_app.logger.error('File error!')
+ return False
+
+ @classmethod
+ def from_json(cls, server):
+
+ try:
+ if cls.check_json_format(json.loads(server)):
+ decoded = json.loads(server, cls=ServerDecoder)
+ return cls(**decoded)
+ except (jsonschema.ValidationError, ValueError) as e:
+ raise e
+
+ @classmethod
+ def from_dict(cls, server):
+ return cls.from_json(json.dumps(server))
+
+ def json(self):
+ return json.dumps(self.__dict__, cls=ServerEncoder)
+
+ @staticmethod
+ def check_json_format(server):
+
+ try:
+ validate(server)
+ except jsonschema.ValidationError as e:
+ raise e
+
+ for key in ('ip', 'ip6'):
+ if key in server:
+ address = ipaddress.ip_address(server[key])
+
+ if any({address.is_private, address.is_multicast,
+ address.is_unspecified, address.is_reserved,
+ address.is_loopback}):
+ raise ValueError('{} is not accepted!\n'.format(key))
+
+ return True
+
+ def save(self):
+
+ filepath = Path('data') / (self.name.lower() + '.json')
+
+ try:
+ with filepath.open(mode='w', encoding='utf-8') as f:
+ json.dump(self.__dict__, f, cls=ServerEncoder)
+ except Exception as e:
+ raise e
+
+ def update_weights(self):
+
+ if self.type not in ('exit', 'relay'):
+ raise NotImplementedError
+
+ url = 'https://onionoo.torproject.org/weights?lookup={}'.format(
+ self.fingerprint
+ )
+
+ data = requests.get(url)
+
+ try:
+ data.raise_for_status()
+ except requests.HTTPError as e:
+ raise e
+ else:
+ data = data.json()['relays'][0]
+
+ self.mean_consensus_weight = calculate_weight(data['consensus_weight'])
+ self.mean_exit_probability = calculate_weight(data['exit_probability'])
+ self.mean_guard_probability = calculate_weight(
+ data['guard_probability']
+ )
+ self.mean_middle_probability = calculate_weight(
+ data['middle_probability']
+ )
+ self.mean_consensus_weight_fraction = calculate_weight(
+ data['consensus_weight_fraction']
+ )
+
+ def check_status(self):
+
+ now = datetime.utcnow()
+ delta = now - self.last_updated
+
+ if delta.seconds >= 3600:
+ self.status = False
+ self.tor_status = False
+ return False
+ else:
+ return True
diff --git a/ennstatus/api/views.py b/ennstatus/api/views.py
--- a/ennstatus/api/views.py
+++ b/ennstatus/api/views.py
@@ -1,13 +1,22 @@
import ipaddress
+import json
+
+from datetime import datetime
from flask import (Blueprint, request, current_app, jsonify, render_template,
abort)
-from ennstatus.api.functions import check_json_format, update_server
+import strict_rfc3339
+import pygeoip
+
from ennstatus.status.functions import (single_server, all_servers,
all_servers_by_type)
+from .model import Server
+
api_page = Blueprint('api', __name__)
+gi4 = pygeoip.GeoIP('/usr/share/GeoIP/GeoIP.dat', pygeoip.MEMORY_CACHE)
+gi6 = pygeoip.GeoIP('/usr/share/GeoIP/GeoIPv6.dat', pygeoip.MEMORY_CACHE)
@api_page.route('/update', methods=('POST',))
@@ -19,21 +28,21 @@ def update():
else:
accepted_ips = current_app.config.get('ENNSTATUS_ACCEPTED_IPS', [])
- json = request.get_json()
- if json is None:
+ if request.remote_addr not in accepted_ips:
+ current_app.logger.warn('Unallowed IP %s tried to update data!'
+ % request.remote_addr)
+ return 'IP not allowed!\n', 403, {'Content-Type': 'text/plain'}
+
+ data = request.get_json()
+
+ if data is None:
current_app.logger.info('No JSON data supplied!')
return 'No JSON data supplied!\n', 400, {'Content-Type': 'text/plain'}
- try:
- check_json_format(json)
- except ValueError as e:
- current_app.logger.warning(' '.join([str(e), str(json)]))
- return str(e), 409, {'Content-Type': 'text/plain'}
-
- if 'ip' in json:
- ip = json['ip']
- elif 'ip6' in json:
- ip = json['ip6']
+ if 'ip' in data:
+ ip = data['ip']
+ elif 'ip6' in data:
+ ip = data['ip6']
else:
ip = request.remote_addr
@@ -42,25 +51,37 @@ def update():
except ipaddress.AddressValueError:
return 'IP not allowed!\n', 403, {'Content-Type': 'text/plain'}
- if request.remote_addr not in accepted_ips:
- current_app.logger.warn('Unallowed IP %s tried to update data!'
- % ip)
- return 'IP not allowed!\n', 403, {'Content-Type': 'text/plain'}
+ if ip.version == 4:
+ data['country'] = gi4.country_name_by_addr(str(ip))
+ elif ip.version == 6:
+ data['country'] = gi6.country_name_by_addr(str(ip))
+ else:
+ data['country'] = None
- current_app.logger.info(str(json))
- server = update_server(server=json, ip=ip)
+ data['last_updated'] = strict_rfc3339.timestamp_to_rfc3339_utcoffset(
+ datetime.utcnow().timestamp()
+ )
+ data['status'] = True
- if server:
- current_app.logger.info('Return result')
- current_app.logger.info(str(server))
- return (jsonify(server), 201,
- {'Location': '/api/export/json/single?server_name=%s'
- % server['server_name']})
+ try:
+ server = Server.from_json(json.dumps(data))
+ except Exception as e:
+ current_app.logger.warning(' '.join([str(e), str(data)]))
+ return str(e), 409, {'Content-Type': 'text/plain'}
+
+ if server.type in ('exit', 'relay'):
+ server.update_weights()
- else:
- current_app.logger.error('Unexpected error: %s' % server,
- exc_info=True)
- return abort(500)
+ try:
+ server.save()
+ except Exception as e:
+ current_app.logger.error(str(e))
+ return str(e), 500, {'Content-Type': 'text/plain'}
+
+ current_app.logger.info('Return result')
+ return (server.json(), 201,
+ {'Location': '/api/export/json/single?server_name=%s'
+ % server.name})
@api_page.route('/export', defaults={'server_type': 'all',
@@ -80,7 +101,8 @@ def export(export_format, server_type):
if server:
if export_format == 'json':
current_app.logger.info('Returning server as json!')
- return jsonify(server)
+ return (server.json(), 200,
+ {'Content-Type': 'application/json'})
else:
current_app.logger.info('Returning server as xml!')
return (
@@ -100,15 +122,14 @@ def export(export_format, server_type):
else:
if server_type == 'all':
current_app.logger.info('Getting all servers!')
- servers = list(all_servers())
+ servers = [server.json() for server in all_servers()]
else:
current_app.logger.info('Getting all %s!' % server_type)
- servers = list(all_servers_by_type(server_type.capitalize()))
+ servers = list(all_servers_by_type(server_type.lower()))
if export_format == 'json':
- response = {'enn_network': servers}
current_app.logger.info('Returning as json!')
- return jsonify(response)
+ return str(servers)
else:
current_app.logger.info('Returning as xml!')
return (render_template('api/export/xml/network.xml',
diff --git a/ennstatus/status/functions.py b/ennstatus/status/functions.py
--- a/ennstatus/status/functions.py
+++ b/ennstatus/status/functions.py
@@ -8,6 +8,7 @@ from datetime import datetime
from flask import current_app
from flask_mail import Mail, Message
+from ..api.model import Server
mail = Mail()
@@ -99,29 +100,33 @@ def _load_single_server(filename):
def single_server(name):
- filename = ''.join(['data/', name, '.json'])
- return _load_single_server(filename)
+ server = Server.from_file_by_name(name)
+
+ if server:
+ server.check_status()
+
+ return server
def _get_json_files(root, files):
for f in files:
if f.endswith('.json'):
- yield os.path.join(root, f)
+ yield f
def all_servers():
for root, _, files in os.walk('data'):
for f in _get_json_files(root, files):
- yield _load_single_server(f)
+ yield single_server(f[:-5])
def all_servers_by_type(type):
for server in all_servers():
try:
- if server['server_type'] == type:
+ if server.type == type:
yield server
except TypeError:
continue
@@ -133,7 +138,7 @@ def split_all_servers_to_types():
for server in all_servers():
try:
- servers[server['server_type']].append(server)
+ servers[server.type].append(server)
except TypeError:
continue
diff --git a/ennstatus/status/views.py b/ennstatus/status/views.py
--- a/ennstatus/status/views.py
+++ b/ennstatus/status/views.py
@@ -13,8 +13,8 @@ def index():
servers = split_all_servers_to_types()
current_app.logger.info('Returning servers')
- return render_template('status/index.html', exit=servers['Exit'],
- relay=servers['Relay'], bridge=servers['Bridge'])
+ return render_template('status/index.html', exit=servers['exit'],
+ relay=servers['relay'], bridge=servers['bridge'])
@status_page.route('/exit')
diff --git a/ennstatus/templates/api/export/xml/server.xml b/ennstatus/templates/api/export/xml/server.xml
--- a/ennstatus/templates/api/export/xml/server.xml
+++ b/ennstatus/templates/api/export/xml/server.xml
@@ -1,20 +1,20 @@
No servers found!
{% else %} {% if exit %} - {{ macros.create_server_table('Exit', exit) }} + {{ macros.create_server_table('exit', exit) }} {% endif %} {% if relay %} - {{ macros.create_server_table('Relay', relay) }} + {{ macros.create_server_table('relay', relay) }} {% endif %} {% if bridge %} - {{ macros.create_server_table('Bridge', bridge) }} + {{ macros.create_server_table('bridge', bridge) }} {% endif %} {% endif %} diff --git a/ennstatus/templates/status/macros.html b/ennstatus/templates/status/macros.html --- a/ennstatus/templates/status/macros.html +++ b/ennstatus/templates/status/macros.html @@ -1,7 +1,7 @@ {% macro colorize_status(status) %} - {% if status == "Online" %} + {% if status %} {% set color = "text-success" %} - {% elif status == "Unknown" %} + {% elif status is none %} {% set color = "text-warning" %} {% else %} {% set color = "text-danger" %} @@ -24,7 +24,7 @@ {% endmacro %} {% macro create_fingerprint(fingerprint, server_type) %} - {% if server_type in ('Exit', 'Relay') %} + {% if server_type in ('exit', 'relay') %} {% set url_type = 'relay' %} {% else %} {% set url_type = 'bridge' %} @@ -49,7 +49,7 @@ {% else %} {% set headers = ('#', 'Name', 'Server Status', 'Tor Status', 'Country', 'Fingerprint', 'OBFS', 'FTEProxy', 'Flashproxy', 'meek', 'Last Updated (UTC)') %} {% endif %} -{{ loop.index }} | - {% if server_type in ('Exit', 'Relay') %} -{{ create_name(server['server_name']) }} | -{{ server['ip'] }} | -{{ server.get('ip6', 'N/A') }} | + {% if server_type in ('exit', 'relay') %} +{{ create_name(server.name) }} | +{{ server.ip }} | +{{ server.ip6 or 'N/A' }} | {% else %} - {% if server['server_name'] in config['ENNSTATUS_BRIDGE_PROGRAM'] %} -{{ server['server_name'] }} | + {% if server.name in config['ENNSTATUS_BRIDGE_PROGRAM'] %} +{{ server.name }} | {% else %} -{{ server['server_name'] }} | +{{ server.name }} | {% endif %} {% endif %} - {% for status in (server['server_status'], server['tor_status']) %} + {% for status in (server.status, server.tor_status) %}{{ colorize_status(status) }} | {% endfor %} -{{ create_country(server['country']) }} | -{{ create_fingerprint(server['fingerprint'], server['server_type']) }} | - {% if server_type == 'Bridge' %} -{{ colorize_obfs(server['obfs']) }} | -{{ colorize_obfs(server['fteproxy']) }} | -{{ colorize_obfs(server['flashproxy']) }} | -{{ colorize_obfs(server['meek']) }} | +{{ create_country(server.country) }} | +{{ create_fingerprint(server.fingerprint, server.type) }} | + {% if server_type == 'bridge' %} +{{ colorize_obfs(server.obfs) }} | +{{ colorize_obfs(server.fteproxy) }} | +{{ colorize_obfs(server.flashproxy) }} | +{{ colorize_obfs(server.meek) }} | {% endif %}{{ server['last_updated'] }} |
See our bridge program, if you want to fund some bridges!