diff --git a/ennstatus/api/model.py b/ennstatus/api/model.py
--- a/ennstatus/api/model.py
+++ b/ennstatus/api/model.py
@@ -24,10 +24,11 @@ from datetime import datetime
import jsonschema
import strict_rfc3339
-import requests
from flask import current_app
from pkg_resources import resource_filename
+from onion_py.manager import Manager
+from onion_py.caching import OnionSimpleCache
from ..utils import check_ip
@@ -45,6 +46,8 @@ validate = functools.partial(
format_checker=jsonschema.FormatChecker()
)
+manager = Manager(OnionSimpleCache())
+
def calculate_weight(data):
@@ -57,9 +60,9 @@ def calculate_weight(data):
except KeyError:
continue
- factor = subdata['factor']
+ factor = subdata.factor
- values = [x * factor for x in subdata['values'] if x is not None]
+ values = [x * factor for x in subdata.values if x is not None]
if values:
obj[subkey] = statistics.mean(values) * 100
@@ -117,6 +120,7 @@ class Server:
self.last_updated = kwargs['last_updated']
self.country = kwargs['country']
self.bandwidth = kwargs.get('bandwidth')
+ self.flags = kwargs.get('flags')
if self.type == 'bridge':
self.obfs = kwargs.get('obfs')
@@ -224,33 +228,37 @@ class Server:
if self.type not in ('exit', 'relay'):
raise NotImplementedError
- url = 'https://onionoo.torproject.org/weights?lookup={}'.format(
- self.fingerprint
+ try:
+ data = manager.query('weights', lookup=self.fingerprint)
+ except:
+ raise NotImplementedError
+
+ if data is not None:
+ data = data.relays[0]
+
+ self.mean_consensus_weight = calculate_weight(data.consensus_weight)
+ self.mean_exit_probability = calculate_weight(data.exit_probability)
+ self.mean_guard_probability = calculate_weight(
+ data.guard_probability
+ )
+ self.mean_middle_probability = calculate_weight(
+ data.middle_probability
+ )
+ self.mean_consensus_weight_fraction = calculate_weight(
+ data.consensus_weight_fraction
)
- data = requests.get(url)
+ def update_flags(self):
try:
- data.raise_for_status()
- except requests.HTTPError as e:
- raise e
- else:
- try:
- data = data.json()['relays'][0]
- except IndexError as e:
- raise RuntimeError from e
+ data = manager.query('details', lookup=self.fingerprint)
+ except:
+ raise NotImplementedError
- self.mean_consensus_weight = calculate_weight(data['consensus_weight'])
- self.mean_exit_probability = calculate_weight(data['exit_probability'])
- self.mean_guard_probability = calculate_weight(
- data['guard_probability']
- )
- self.mean_middle_probability = calculate_weight(
- data['middle_probability']
- )
- self.mean_consensus_weight_fraction = calculate_weight(
- data['consensus_weight_fraction']
- )
+ if data is not None:
+ self.flags = data.relays[0].flags
+ else:
+ raise NotImplementedError
def check_status(self):
diff --git a/ennstatus/api/schema/server.json b/ennstatus/api/schema/server.json
--- a/ennstatus/api/schema/server.json
+++ b/ennstatus/api/schema/server.json
@@ -100,6 +100,11 @@
},
"type": {
"type": "string"
+ },
+ "flags": {
+ "type": "array",
+ "items": { "type": "string" },
+ "uniqueItems": true
}
},
"required": [
diff --git a/ennstatus/api/views.py b/ennstatus/api/views.py
--- a/ennstatus/api/views.py
+++ b/ennstatus/api/views.py
@@ -26,7 +26,6 @@ from werkzeug.exceptions import BadReque
import strict_rfc3339
import pygeoip
-import requests
from ennstatus import csrf
from ennstatus.status.functions import (single_server, all_servers,
@@ -120,11 +119,10 @@ def update():
server.update_weights()
except NotImplementedError:
pass
- except requests.HTTPError as e:
- current_app.logger.error(str(e), exc_info=True)
- pass
- except RuntimeError as e:
- current_app.logger.error(str(e), exc_info=True)
+
+ try:
+ server.update_flags()
+ except NotImplementedError:
pass
try:
diff --git a/ennstatus/cli/__init__.py b/ennstatus/cli/__init__.py
--- a/ennstatus/cli/__init__.py
+++ b/ennstatus/cli/__init__.py
@@ -15,9 +15,13 @@
# along with this program. If not, see .
import pathlib
+import importlib
+import operator
import click
+from .commands import __all__ as commands_all
+
@click.group()
@click.option('-p', '--path', default='/srv/http/enn.lu',
@@ -35,9 +39,10 @@ def cli(ctx, path):
ctx.obj['config_file'] = path / 'config.json'
ctx.obj['data_dir'] = path / 'data'
-
-from .commands import config
-cli.add_command(config, 'config')
+subcommands = importlib.import_module('ennstatus.cli.commands')
+for command in commands_all:
+ get = operator.attrgetter(command)
+ cli.add_command(get(subcommands), command)
if __name__ == '__main__':
cli()
diff --git a/ennstatus/cli/commands/__init__.py b/ennstatus/cli/commands/__init__.py
--- a/ennstatus/cli/commands/__init__.py
+++ b/ennstatus/cli/commands/__init__.py
@@ -15,5 +15,6 @@
# along with this program. If not, see .
from .config import config
+from .stats import stats
-__all__ = ['config']
+__all__ = ['config', 'stats']
diff --git a/ennstatus/cli/commands/config.py b/ennstatus/cli/commands/config.py
--- a/ennstatus/cli/commands/config.py
+++ b/ennstatus/cli/commands/config.py
@@ -16,6 +16,7 @@
import json
import ipaddress
+import subprocess
from pprint import pprint
@@ -83,6 +84,7 @@ def add(obj, name, ips, password, bridge
except KeyError:
config['ENNSTATUS_BRIDGE_PROGRAM'] = [name]
+ click.echo('%s password: %s' % (name, password))
with obj['config_file'].open(mode='w', encoding='utf-8') as f:
json.dump(config, f, indent=4, sort_keys=True)
@@ -103,7 +105,7 @@ def delete(obj, name):
try:
config['ENNSTATUS_BRIDGE_PROGRAM'].remove(name)
- except KeyError:
+ except (KeyError, ValueError):
pass
with obj['config_file'].open(mode='w', encoding='utf-8') as f:
@@ -114,3 +116,37 @@ def delete(obj, name):
filename.unlink()
except FileNotFoundError:
pass
+
+
+@config.group(short_help='Configure more servers at once')
+def servers():
+ pass
+
+
+@servers.command('add', short_help='Add servers')
+@click.argument('names', nargs=-1, required=True)
+@click.option('-i', '--ips', prompt='IPs (comma separated)')
+@click.option('--bridgeprogram/--no-bridgeprogram', default=False)
+@click.pass_context
+def adds(ctx, names, ips, bridgeprogram):
+
+ for name in names:
+ password = subprocess.check_output(['pwgen', '8', '1'])
+ password = password.decode('utf-8')
+ password = password.strip()
+ ctx.invoke(
+ add,
+ name=name,
+ ips=ips,
+ password=password,
+ bridgeprogram=bridgeprogram
+ )
+
+
+@servers.command('delete', short_help='Delete servers')
+@click.argument('names', nargs=-1, required=True)
+@click.pass_context
+def dels(ctx, names):
+
+ for name in names:
+ ctx.invoke(delete, name=name)
diff --git a/ennstatus/cli/commands/stats.py b/ennstatus/cli/commands/stats.py
new file mode 100644
--- /dev/null
+++ b/ennstatus/cli/commands/stats.py
@@ -0,0 +1,224 @@
+import json
+
+from collections import defaultdict
+
+import click
+
+from ennstatus import create_app
+from ...status.functions import split_all_servers_to_types
+
+
+@click.group(short_help='Get statistics')
+def stats():
+ pass
+
+
+@stats.command('count')
+@click.option('--by-type', 'by_type', is_flag=True, default=False)
+@click.pass_obj
+def count(obj, by_type):
+
+ def calculate_host_number(config, type='all', servers=None):
+
+ hosts = set()
+ if type == 'all':
+
+ for values in config['ENNSTATUS_SERVERS'].values():
+ ips = frozenset(values['IPS'])
+ hosts.add(ips)
+
+ return len(hosts)
+ else:
+ for server in servers[servertype]:
+ ips = frozenset(
+ config['ENNSTATUS_SERVERS'][server.name.lower()]['IPS']
+ )
+ hosts.add(ips)
+ return len(hosts)
+
+ app = create_app()
+
+ with app.app_context():
+ app.logger.disabled = True
+ servers = split_all_servers_to_types()
+
+ if not by_type:
+
+ click.echo(
+ 'We have %s servers in total!' % (
+ click.style(
+ str(sum(len(x) for x in servers.values())),
+ fg='blue'
+ )
+ )
+ )
+
+ click.echo(
+ 'We have %s different hosts!' % (
+ click.style(
+ str(calculate_host_number(app.config)),
+ fg='blue'
+ )
+ )
+ )
+ else:
+ for servertype, server in servers.items():
+ click.echo(
+ 'We have %s %s servers!' % (
+ click.style(
+ str(len(server)),
+ fg='blue'
+ ),
+ click.style(
+ servertype,
+ fg='red'
+ )
+ )
+ )
+ click.echo(
+ 'We have %s different %s hosts!' % (
+ click.style(
+ str(
+ calculate_host_number(
+ app.config,
+ type=servertype,
+ servers=servers
+ )
+ ),
+ fg='blue'
+ ),
+ click.style(
+ servertype,
+ fg='red'
+ )
+ )
+ )
+
+
+@stats.command('countries')
+@click.option('--by-type', 'by_type', is_flag=True, default=False)
+@click.pass_obj
+def countries(obj, by_type):
+ app = create_app()
+
+ with app.app_context():
+ app.logger.disabled = True
+ servers = split_all_servers_to_types()
+
+ if not by_type:
+ countries = defaultdict(int)
+
+ for key, value in servers.items():
+ for server in value:
+ countries[server.country] += 1
+
+ for key, value in sorted(countries.items(), key=lambda x: x[1]):
+ click.echo(
+ '%s: %s' % (
+ click.style(key, fg='green'),
+ click.style(str(value), fg='blue')
+ )
+ )
+
+ click.echo(
+ 'We are hosted in %s different countries' % click.style(
+ str(len(countries.keys())),
+ fg='blue'
+ )
+ )
+ else:
+ type_countries = {
+ 'exit': defaultdict(int),
+ 'relay': defaultdict(int),
+ 'bridge': defaultdict(int)
+ }
+
+ for key, value in servers.items():
+ for server in value:
+ type_countries[key][server.country] += 1
+
+ type_countries = dict((k, v) for k, v in type_countries.items() if v)
+
+ for key, value in sorted(type_countries.items(), key=lambda x: x[0]):
+ click.echo(
+ click.style(
+ key.capitalize(),
+ fg='red',
+ bold=True,
+ underline=True
+ )
+ )
+
+ for country, count in sorted(type_countries[key].items(), key=lambda x: x[1]):
+ click.echo(
+ '%s: %s' % (
+ click.style(country, fg='green'),
+ click.style(str(count), fg='blue')
+ )
+ )
+
+ click.echo(
+ '%s are hosted in %s different countries' % (
+ key,
+ click.style(
+ str(len(type_countries[key].keys())),
+ fg='blue'
+ )
+ )
+ )
+
+
+@stats.command('exit_probability')
+@click.option('--by-server', 'by_server', is_flag=True, default=False)
+@click.pass_obj
+def exit_probability(obj, by_server):
+
+ app = create_app()
+ with app.app_context():
+ app.logger.disabled = True
+ servers = split_all_servers_to_types()
+
+ if not by_server:
+ exit_probability = defaultdict(int)
+ for server in servers['exit']:
+ for subkey in ('1_week', '1_month', '3_months', '1_year', '5_years'):
+ if server.mean_exit_probability[subkey] is not None:
+ exit_probability[subkey] += server.mean_exit_probability[subkey]
+
+ for subkey in ('1_week', '1_month', '3_months', '1_year', '5_years'):
+ click.echo(
+ 'Mean exit probability over %s: %s' % (
+ click.style(
+ subkey,
+ fg='blue'
+ ),
+ click.style(
+ str(round(exit_probability[subkey], 2)) + '%',
+ fg='red'
+ )
+ )
+ )
+ else:
+ for server in servers['exit']:
+ click.echo(
+ click.style(
+ server.name.capitalize(),
+ fg='red',
+ bold=True,
+ underline=True
+ )
+ )
+ for subkey in ('1_week', '1_month', '3_months', '1_year', '5_years'):
+ if server.mean_exit_probability[subkey] is not None:
+ click.echo(
+ 'Mean exit probabilty over %s: %s' % (
+ click.style(
+ subkey,
+ fg='blue'
+ ),
+ click.style(
+ str(round(server.mean_exit_probability[subkey], 2)) + "%",
+ fg='red'
+ )
+ )
+ )
diff --git a/requirements.in b/requirements.in
--- a/requirements.in
+++ b/requirements.in
@@ -9,5 +9,5 @@ Flask
jsonschema
pygeoip
python-gnupg
-requests
strict-rfc3339
+OnionPy
diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -18,10 +18,11 @@ itsdangerous==0.24 # via flask
Jinja2==2.8 # via flask
jsonschema==2.5.1
MarkupSafe==0.23 # via jinja2
+onionpy==0.3.2
pygeoip==0.3.2
python-gnupg==0.3.8
pytz==2015.7 # via babel
-requests==2.9.1
+requests==2.9.1 # via onionpy
strict-rfc3339==0.6
visitor==0.1.2 # via flask-bootstrap
Werkzeug==0.11.4 # via flask, flask-wtf
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -11,7 +11,7 @@ def _get_requirements():
setup(name='Ennstatus',
- version='5.4.4',
+ version='5.4.5',
description=('Ennstatus provides the user with vital information about '
'the status of the organizations Tor servers.'),
author='Frënn vun der Ënn',