# HG changeset patch
# User Dennis Fink <dennis.fink@c3l.lu>
# Date 2016-05-16 14:27:52
# Node ID 0ef190ed951e5955b2f527ed2f71b20fd45dfe36
# Parent  83bd05c310e2e5460942f81fed58b7d3ee267f93
# Parent  b42ef7e3ca9908def2887040131d98b4a4fa195b

Merged dev

diff --git a/ennstatus/api/model.py b/ennstatus/api/model.py
--- a/ennstatus/api/model.py
+++ b/ennstatus/api/model.py
@@ -24,10 +24,11 @@ from datetime import datetime
 
 import jsonschema
 import strict_rfc3339
-import requests
 
 from flask import current_app
 from pkg_resources import resource_filename
+from onion_py.manager import Manager
+from onion_py.caching import OnionSimpleCache
 
 from ..utils import check_ip
 
@@ -45,6 +46,8 @@ validate = functools.partial(
     format_checker=jsonschema.FormatChecker()
 )
 
+manager = Manager(OnionSimpleCache())
+
 
 def calculate_weight(data):
 
@@ -57,9 +60,9 @@ def calculate_weight(data):
         except KeyError:
             continue
 
-        factor = subdata['factor']
+        factor = subdata.factor
 
-        values = [x * factor for x in subdata['values'] if x is not None]
+        values = [x * factor for x in subdata.values if x is not None]
 
         if values:
             obj[subkey] = statistics.mean(values) * 100
@@ -117,6 +120,7 @@ class Server:
         self.last_updated = kwargs['last_updated']
         self.country = kwargs['country']
         self.bandwidth = kwargs.get('bandwidth')
+        self.flags = kwargs.get('flags')
 
         if self.type == 'bridge':
             self.obfs = kwargs.get('obfs')
@@ -224,33 +228,37 @@ class Server:
         if self.type not in ('exit', 'relay'):
             raise NotImplementedError
 
-        url = 'https://onionoo.torproject.org/weights?lookup={}'.format(
-            self.fingerprint
+        try:
+            data = manager.query('weights', lookup=self.fingerprint)
+        except:
+            raise NotImplementedError
+
+        if data is not None:
+            data = data.relays[0]
+
+        self.mean_consensus_weight = calculate_weight(data.consensus_weight)
+        self.mean_exit_probability = calculate_weight(data.exit_probability)
+        self.mean_guard_probability = calculate_weight(
+            data.guard_probability
+        )
+        self.mean_middle_probability = calculate_weight(
+            data.middle_probability
+        )
+        self.mean_consensus_weight_fraction = calculate_weight(
+            data.consensus_weight_fraction
         )
 
-        data = requests.get(url)
+    def update_flags(self):
 
         try:
-            data.raise_for_status()
-        except requests.HTTPError as e:
-            raise e
-        else:
-            try:
-                data = data.json()['relays'][0]
-            except IndexError as e:
-                raise RuntimeError from e
+            data = manager.query('details', lookup=self.fingerprint)
+        except:
+            raise NotImplementedError
 
-        self.mean_consensus_weight = calculate_weight(data['consensus_weight'])
-        self.mean_exit_probability = calculate_weight(data['exit_probability'])
-        self.mean_guard_probability = calculate_weight(
-            data['guard_probability']
-        )
-        self.mean_middle_probability = calculate_weight(
-            data['middle_probability']
-        )
-        self.mean_consensus_weight_fraction = calculate_weight(
-            data['consensus_weight_fraction']
-        )
+        if data is not None:
+            self.flags = data.relays[0].flags
+        else:
+            raise NotImplementedError
 
     def check_status(self):
 
diff --git a/ennstatus/api/schema/server.json b/ennstatus/api/schema/server.json
--- a/ennstatus/api/schema/server.json
+++ b/ennstatus/api/schema/server.json
@@ -100,6 +100,11 @@
         },
         "type": {
             "type": "string"
+        },
+        "flags": {
+            "type": "array",
+            "items": { "type": "string" },
+            "uniqueItems": true
         }
     },
     "required": [
diff --git a/ennstatus/api/views.py b/ennstatus/api/views.py
--- a/ennstatus/api/views.py
+++ b/ennstatus/api/views.py
@@ -26,7 +26,6 @@ from werkzeug.exceptions import BadReque
 
 import strict_rfc3339
 import pygeoip
-import requests
 
 from ennstatus import csrf
 from ennstatus.status.functions import (single_server, all_servers,
@@ -120,11 +119,10 @@ def update():
         server.update_weights()
     except NotImplementedError:
         pass
-    except requests.HTTPError as e:
-        current_app.logger.error(str(e), exc_info=True)
-        pass
-    except RuntimeError as e:
-        current_app.logger.error(str(e), exc_info=True)
+
+    try:
+        server.update_flags()
+    except NotImplementedError:
         pass
 
     try:
diff --git a/ennstatus/cli/__init__.py b/ennstatus/cli/__init__.py
--- a/ennstatus/cli/__init__.py
+++ b/ennstatus/cli/__init__.py
@@ -15,9 +15,13 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import pathlib
+import importlib
+import operator
 
 import click
 
+from .commands import __all__ as commands_all
+
 
 @click.group()
 @click.option('-p', '--path', default='/srv/http/enn.lu',
@@ -35,9 +39,10 @@ def cli(ctx, path):
     ctx.obj['config_file'] = path / 'config.json'
     ctx.obj['data_dir'] = path / 'data'
 
-
-from .commands import config
-cli.add_command(config, 'config')
+subcommands = importlib.import_module('ennstatus.cli.commands')
+for command in commands_all:
+    get = operator.attrgetter(command)
+    cli.add_command(get(subcommands), command)
 
 if __name__ == '__main__':
     cli()
diff --git a/ennstatus/cli/commands/__init__.py b/ennstatus/cli/commands/__init__.py
--- a/ennstatus/cli/commands/__init__.py
+++ b/ennstatus/cli/commands/__init__.py
@@ -15,5 +15,6 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from .config import config
+from .stats import stats
 
-__all__ = ['config']
+__all__ = ['config', 'stats']
diff --git a/ennstatus/cli/commands/config.py b/ennstatus/cli/commands/config.py
--- a/ennstatus/cli/commands/config.py
+++ b/ennstatus/cli/commands/config.py
@@ -16,6 +16,7 @@
 
 import json
 import ipaddress
+import subprocess
 
 from pprint import pprint
 
@@ -83,6 +84,7 @@ def add(obj, name, ips, password, bridge
         except KeyError:
             config['ENNSTATUS_BRIDGE_PROGRAM'] = [name]
 
+    click.echo('%s password: %s' % (name, password))
     with obj['config_file'].open(mode='w', encoding='utf-8') as f:
         json.dump(config, f, indent=4, sort_keys=True)
 
@@ -103,7 +105,7 @@ def delete(obj, name):
 
     try:
         config['ENNSTATUS_BRIDGE_PROGRAM'].remove(name)
-    except KeyError:
+    except (KeyError, ValueError):
         pass
 
     with obj['config_file'].open(mode='w', encoding='utf-8') as f:
@@ -114,3 +116,37 @@ def delete(obj, name):
         filename.unlink()
     except FileNotFoundError:
         pass
+
+
+@config.group(short_help='Configure more servers at once')
+def servers():
+    pass
+
+
+@servers.command('add', short_help='Add servers')
+@click.argument('names', nargs=-1, required=True)
+@click.option('-i', '--ips', prompt='IPs (comma separated)')
+@click.option('--bridgeprogram/--no-bridgeprogram', default=False)
+@click.pass_context
+def adds(ctx, names, ips, bridgeprogram):
+
+    for name in names:
+        password = subprocess.check_output(['pwgen', '8', '1'])
+        password = password.decode('utf-8')
+        password = password.strip()
+        ctx.invoke(
+            add,
+            name=name,
+            ips=ips,
+            password=password,
+            bridgeprogram=bridgeprogram
+        )
+
+
+@servers.command('delete', short_help='Delete servers')
+@click.argument('names', nargs=-1, required=True)
+@click.pass_context
+def dels(ctx, names):
+
+    for name in names:
+        ctx.invoke(delete, name=name)
diff --git a/ennstatus/cli/commands/stats.py b/ennstatus/cli/commands/stats.py
new file mode 100644
--- /dev/null
+++ b/ennstatus/cli/commands/stats.py
@@ -0,0 +1,224 @@
+import json
+
+from collections import defaultdict
+
+import click
+
+from ennstatus import create_app
+from ...status.functions import split_all_servers_to_types
+
+
+@click.group(short_help='Get statistics')
+def stats():
+    pass
+
+
+@stats.command('count')
+@click.option('--by-type', 'by_type', is_flag=True, default=False)
+@click.pass_obj
+def count(obj, by_type):
+
+    def calculate_host_number(config, type='all', servers=None):
+
+        hosts = set()
+        if type == 'all':
+
+            for values in config['ENNSTATUS_SERVERS'].values():
+                ips = frozenset(values['IPS'])
+                hosts.add(ips)
+
+            return len(hosts)
+        else:
+            for server in servers[servertype]:
+                ips = frozenset(
+                    config['ENNSTATUS_SERVERS'][server.name.lower()]['IPS']
+                )
+                hosts.add(ips)
+        return len(hosts)
+
+    app = create_app()
+
+    with app.app_context():
+        app.logger.disabled = True
+        servers = split_all_servers_to_types()
+
+    if not by_type:
+
+        click.echo(
+            'We have %s servers in total!' % (
+                click.style(
+                    str(sum(len(x) for x in servers.values())),
+                    fg='blue'
+                )
+            )
+        )
+
+        click.echo(
+            'We have %s different hosts!' % (
+                click.style(
+                    str(calculate_host_number(app.config)),
+                    fg='blue'
+                )
+            )
+        )
+    else:
+        for servertype, server in servers.items():
+            click.echo(
+                'We have %s %s servers!' % (
+                    click.style(
+                        str(len(server)),
+                        fg='blue'
+                    ),
+                    click.style(
+                        servertype,
+                        fg='red'
+                    )
+                )
+            )
+            click.echo(
+                'We have %s different %s hosts!' % (
+                    click.style(
+                        str(
+                            calculate_host_number(
+                                app.config,
+                                type=servertype,
+                                servers=servers
+                            )
+                        ),
+                        fg='blue'
+                    ),
+                    click.style(
+                        servertype,
+                        fg='red'
+                    )
+                )
+            )
+
+
+@stats.command('countries')
+@click.option('--by-type', 'by_type', is_flag=True, default=False)
+@click.pass_obj
+def countries(obj, by_type):
+    app = create_app()
+
+    with app.app_context():
+        app.logger.disabled = True
+        servers = split_all_servers_to_types()
+
+    if not by_type:
+        countries = defaultdict(int)
+
+        for key, value in servers.items():
+            for server in value:
+                countries[server.country] += 1
+
+        for key, value in sorted(countries.items(), key=lambda x: x[1]):
+            click.echo(
+                '%s: %s' % (
+                    click.style(key, fg='green'),
+                    click.style(str(value), fg='blue')
+                )
+            )
+
+        click.echo(
+            'We are hosted in %s different countries' % click.style(
+                str(len(countries.keys())),
+                fg='blue'
+            )
+        )
+    else:
+        type_countries = {
+            'exit': defaultdict(int),
+            'relay': defaultdict(int),
+            'bridge': defaultdict(int)
+        }
+
+        for key, value in servers.items():
+            for server in value:
+                type_countries[key][server.country] += 1
+
+        type_countries = dict((k, v) for k, v in type_countries.items() if v)
+
+        for key, value in sorted(type_countries.items(), key=lambda x: x[0]):
+            click.echo(
+                click.style(
+                    key.capitalize(),
+                    fg='red',
+                    bold=True,
+                    underline=True
+                )
+            )
+
+            for country, count in sorted(type_countries[key].items(), key=lambda x: x[1]):
+                click.echo(
+                    '%s: %s' % (
+                        click.style(country, fg='green'),
+                        click.style(str(count), fg='blue')
+                    )
+                )
+
+            click.echo(
+                '%s are hosted in %s different countries' % (
+                    key,
+                    click.style(
+                        str(len(type_countries[key].keys())),
+                        fg='blue'
+                    )
+                )
+            )
+
+
+@stats.command('exit_probability')
+@click.option('--by-server', 'by_server', is_flag=True, default=False)
+@click.pass_obj
+def exit_probability(obj, by_server):
+
+    app = create_app()
+    with app.app_context():
+        app.logger.disabled = True
+        servers = split_all_servers_to_types()
+
+    if not by_server:
+        exit_probability = defaultdict(int)
+        for server in servers['exit']:
+            for subkey in ('1_week', '1_month', '3_months', '1_year', '5_years'):
+                if server.mean_exit_probability[subkey] is not None:
+                    exit_probability[subkey] += server.mean_exit_probability[subkey]
+
+        for subkey in ('1_week', '1_month', '3_months', '1_year', '5_years'):
+            click.echo(
+                'Mean exit probability over %s: %s' % (
+                    click.style(
+                        subkey,
+                        fg='blue'
+                    ),
+                    click.style(
+                        str(round(exit_probability[subkey], 2)) + '%',
+                        fg='red'
+                    )
+                )
+            )
+    else:
+        for server in servers['exit']:
+            click.echo(
+                click.style(
+                    server.name.capitalize(),
+                    fg='red',
+                    bold=True,
+                    underline=True
+                )
+            )
+            for subkey in ('1_week', '1_month', '3_months', '1_year', '5_years'):
+                if server.mean_exit_probability[subkey] is not None:
+                    click.echo(
+                        'Mean exit probabilty over %s: %s' % (
+                            click.style(
+                                subkey,
+                                fg='blue'
+                            ),
+                            click.style(
+                                str(round(server.mean_exit_probability[subkey], 2)) + "%",
+                                fg='red'
+                            )
+                        )
+                    )
diff --git a/requirements.in b/requirements.in
--- a/requirements.in
+++ b/requirements.in
@@ -9,5 +9,5 @@ Flask
 jsonschema
 pygeoip
 python-gnupg
-requests
 strict-rfc3339
+OnionPy
diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -18,10 +18,11 @@ itsdangerous==0.24        # via flask
 Jinja2==2.8               # via flask
 jsonschema==2.5.1
 MarkupSafe==0.23          # via jinja2
+onionpy==0.3.2
 pygeoip==0.3.2
 python-gnupg==0.3.8
 pytz==2015.7              # via babel
-requests==2.9.1
+requests==2.9.1           # via onionpy
 strict-rfc3339==0.6
 visitor==0.1.2            # via flask-bootstrap
 Werkzeug==0.11.4          # via flask, flask-wtf
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -11,7 +11,7 @@ def _get_requirements():
 
 
 setup(name='Ennstatus',
-      version='5.4.4',
+      version='5.4.5',
       description=('Ennstatus provides the user with vital information about '
                    'the status of the organizations Tor servers.'),
       author='Frënn vun der Ënn',