FVDE/ennstatus Changeset - 0ef190ed951e · Chaos Computer Club Lëtzebuerg Projects

Changeset - 0ef190ed951e

Parent rev.

Child rev.

[Not reviewed]

Merge default

0 9 1

Dennis Fink - 9 years ago 2016-05-16 14:27:52
dennis.fink@c3l.lu

Merged dev

10 files changed with 317 insertions and 39 deletions:

ennstatus/api/model.py

ennstatus/api/schema/server.json

ennstatus/api/views.py

ennstatus/cli/__init__.py

ennstatus/cli/commands/__init__.py

ennstatus/cli/commands/config.py

ennstatus/cli/commands/stats.py

224

requirements.in

requirements.txt

setup.py

0 comments (0 inline, 0 general)

ennstatus/api/model.py

➞

Show inline comments

@@ @@ -15,60 +15,63 @@ @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import ipaddress
 import json
 import functools
 import statistics
 from pathlib import Path
 from datetime import datetime
 import jsonschema
 import strict_rfc3339
 import requests
 from flask import current_app
 from pkg_resources import resource_filename
 from onion_py.manager import Manager
 from onion_py.caching import OnionSimpleCache
 from ..utils import check_ip
 schema = json.load(
     open(
         resource_filename('ennstatus.api', 'schema/server.json'),
         encoding='utf-8'
+    )
+)
 validate = functools.partial(
     jsonschema.validate,
     schema=schema,
     format_checker=jsonschema.FormatChecker()
+)
 manager = Manager(OnionSimpleCache())
 def calculate_weight(data):
     obj = {}
     for subkey in ('1_week', '1_month', '3_months', '1_year', '5_years'):
         try:
             subdata = data[subkey]
         except KeyError:
             continue
-        factor = subdata['factor']
+        factor = subdata.factor
-        values = [x * factor for x in subdata['values'] if x is not None]
+        values = [x * factor for x in subdata.values if x is not None]
         if values:
             obj[subkey] = statistics.mean(values) * 100
         else:
             obj[subkey] = None
     return obj
 class ServerEncoder(json.JSONEncoder):
     def default(self, obj):
@@ @@ -108,24 +111,25 @@ class ServerDecoder(json.JSONDecoder): @@
 class Server:
     def __init__(self, *args, **kwargs):
         self.name = kwargs['name']
         self.type = kwargs['type']
         self.status = kwargs.get('status')
         self.fingerprint = kwargs['fingerprint']
         self.last_updated = kwargs['last_updated']
         self.country = kwargs['country']
         self.bandwidth = kwargs.get('bandwidth')
         self.flags = kwargs.get('flags')
         if self.type == 'bridge':
             self.obfs = kwargs.get('obfs')
             self.fteproxy = kwargs.get('fteproxy')
             self.flashproxy = kwargs.get('flashproxy')
             self.meek = kwargs.get('meek')
         else:
             self.ip = kwargs['ip']
             if 'ip6' in kwargs:
                 self.ip6 = kwargs['ip6']
@@ @@ -215,49 +219,53 @@ class Server: @@
         try:
             with filepath.open(mode='w', encoding='utf-8') as f:
                 json.dump(self.__dict__, f, cls=ServerEncoder)
         except Exception as e:
             raise e
     def update_weights(self):
         if self.type not in ('exit', 'relay'):
             raise NotImplementedError
         url = 'https://onionoo.torproject.org/weights?lookup={}'.format(
             self.fingerprint
         try:
             data = manager.query('weights', lookup=self.fingerprint)
         except:
             raise NotImplementedError
         if data is not None:
             data = data.relays[0]
         self.mean_consensus_weight = calculate_weight(data.consensus_weight)
         self.mean_exit_probability = calculate_weight(data.exit_probability)
         self.mean_guard_probability = calculate_weight(
             data.guard_probability
+        )
         self.mean_middle_probability = calculate_weight(
             data.middle_probability
+        )
         self.mean_consensus_weight_fraction = calculate_weight(
             data.consensus_weight_fraction
+        )
-        data = requests.get(url)
+    def update_flags(self):
         try:
             data.raise_for_status()
         except requests.HTTPError as e:
             raise e
         else:
             try:
                 data = data.json()['relays'][0]
             except IndexError as e:
                 raise RuntimeError from e
             data = manager.query('details', lookup=self.fingerprint)
         except:
             raise NotImplementedError
         self.mean_consensus_weight = calculate_weight(data['consensus_weight'])
         self.mean_exit_probability = calculate_weight(data['exit_probability'])
         self.mean_guard_probability = calculate_weight(
             data['guard_probability']
+        )
         self.mean_middle_probability = calculate_weight(
             data['middle_probability']
+        )
         self.mean_consensus_weight_fraction = calculate_weight(
             data['consensus_weight_fraction']
+        )
         if data is not None:
             self.flags = data.relays[0].flags
         else:
             raise NotImplementedError
     def check_status(self):
         now = datetime.utcnow()
         delta = now - self.last_updated
         if delta.seconds >= 3600:
             self.status = False
         elif delta.seconds >= 600:
             self.status = None

ennstatus/api/schema/server.json

➞

Show inline comments

@@ @@ -91,24 +91,29 @@ @@
             "type": "string",
             "format": "date-time"
         },
         "bandwidth": {
             "type": [
                 "string",
                 "null"
             ],
             "default": null
         },
         "type": {
             "type": "string"
         },
         "flags": {
             "type": "array",
             "items": { "type": "string" },
             "uniqueItems": true
+        }
     },
     "required": [
         "name",
         "status",
         "last_updated",
         "country",
         "fingerprint",
         "type"
     ],
     "definitions": {
         "weights": {

ennstatus/api/views.py

➞

Show inline comments

@@ @@ -17,25 +17,24 @@ @@
 import ipaddress
 import json
 from datetime import datetime
 from flask import (Blueprint, request, current_app, jsonify, render_template,
                    abort)
 from werkzeug.exceptions import BadRequest
 import strict_rfc3339
 import pygeoip
 import requests
 from ennstatus import csrf
 from ennstatus.status.functions import (single_server, all_servers,
                                         all_servers_by_type)
 from .model import Server
 from .auth import httpauth
 api_page = Blueprint('api', __name__)
 gi4 = pygeoip.GeoIP('/usr/share/GeoIP/GeoIP.dat', pygeoip.MEMORY_CACHE)
 gi6 = pygeoip.GeoIP('/usr/share/GeoIP/GeoIPv6.dat', pygeoip.MEMORY_CACHE)
@@ @@ -111,29 +110,28 @@ def update(): @@
+    )
     try:
         server = Server.from_dict(data)
     except Exception as e:
         current_app.logger.warning(' '.join([str(e), str(data)]))
         return str(e), 409, {'Content-Type': 'text/plain'}
     try:
         server.update_weights()
     except NotImplementedError:
         pass
     except requests.HTTPError as e:
         current_app.logger.error(str(e), exc_info=True)
         pass
     except RuntimeError as e:
         current_app.logger.error(str(e), exc_info=True)
     try:
         server.update_flags()
     except NotImplementedError:
         pass
     try:
         server.save()
     except Exception as e:
         current_app.logger.error(str(e))
         return str(e), 500, {'Content-Type': 'text/plain'}
     current_app.logger.info('Return result')
     return (
         server.json(), 201,
+        {

ennstatus/cli/__init__.py

➞

Show inline comments

@@ @@ -6,38 +6,43 @@ @@
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import pathlib
 import importlib
 import operator
 import click
 from .commands import __all__ as commands_all
 @click.group()
 @click.option('-p', '--path', default='/srv/http/enn.lu',
               help='Path where the config files are',
               type=click.Path(exists=True,
                               file_okay=False,
                               writable=True,
                               readable=True)
+              )
 @click.pass_context
 def cli(ctx, path):
     ctx.obj = {}
     path = pathlib.Path(path)
     ctx.obj['path'] = path
     ctx.obj['config_file'] = path / 'config.json'
     ctx.obj['data_dir'] = path / 'data'
 from .commands import config
 cli.add_command(config, 'config')
 subcommands = importlib.import_module('ennstatus.cli.commands')
 for command in commands_all:
     get = operator.attrgetter(command)
     cli.add_command(get(subcommands), command)
 if __name__ == '__main__':
     cli()

ennstatus/cli/commands/__init__.py

➞

Show inline comments

@@ @@ -6,14 +6,15 @@ @@
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 from .config import config
 from .stats import stats
 __all__ = ['config']
+__all__ = ['config', 'stats']

ennstatus/cli/commands/config.py

➞

Show inline comments

@@ @@ -7,24 +7,25 @@ @@
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import json
 import ipaddress
 import subprocess
 from pprint import pprint
 import click
 from ...utils import check_ip
 @click.group(short_help='Configure ennstatus')
 def config():
     pass
@@ @@ -74,43 +75,78 @@ def add(obj, name, ips, password, bridge @@
     config['ENNSTATUS_SERVERS'][name] = {
         'IPS': ips,
         'PASSWORD': password
+    }
     if bridgeprogram:
         try:
             config['ENNSTATUS_BRIDGE_PROGRAM'].append(name)
         except KeyError:
             config['ENNSTATUS_BRIDGE_PROGRAM'] = [name]
     click.echo('%s password: %s' % (name, password))
     with obj['config_file'].open(mode='w', encoding='utf-8') as f:
         json.dump(config, f, indent=4, sort_keys=True)
 @server.command('delete', short_help='Remove server')
 @click.argument('name')
 @click.pass_obj
 def delete(obj, name):
     with obj['config_file'].open() as f:
         config = json.load(f)
     name = name.lower()
     try:
         del config['ENNSTATUS_SERVERS'][name]
     except KeyError:
         raise SystemExit('{} does not exists!'.format(name))
     try:
         config['ENNSTATUS_BRIDGE_PROGRAM'].remove(name)
     except KeyError:
+    except (KeyError, ValueError):
         pass
     with obj['config_file'].open(mode='w', encoding='utf-8') as f:
         json.dump(config, f, indent=4, sort_keys=True)
     filename = obj['data_dir'] / (name + '.json')
     try:
         filename.unlink()
     except FileNotFoundError:
         pass
 @config.group(short_help='Configure more servers at once')
 def servers():
     pass
 @servers.command('add', short_help='Add servers')
 @click.argument('names', nargs=-1, required=True)
 @click.option('-i', '--ips', prompt='IPs (comma separated)')
 @click.option('--bridgeprogram/--no-bridgeprogram', default=False)
 @click.pass_context
 def adds(ctx, names, ips, bridgeprogram):
     for name in names:
         password = subprocess.check_output(['pwgen', '8', '1'])
         password = password.decode('utf-8')
         password = password.strip()
         ctx.invoke(
             add,
             name=name,
             ips=ips,
             password=password,
             bridgeprogram=bridgeprogram
+        )
 @servers.command('delete', short_help='Delete servers')
 @click.argument('names', nargs=-1, required=True)
 @click.pass_context
 def dels(ctx, names):
     for name in names:
         ctx.invoke(delete, name=name)

ennstatus/cli/commands/stats.py

➞

Show inline comments

@@ new file 100644 @@
 import json
 from collections import defaultdict
 import click
 from ennstatus import create_app
 from ...status.functions import split_all_servers_to_types
 @click.group(short_help='Get statistics')
 def stats():
     pass
 @stats.command('count')
 @click.option('--by-type', 'by_type', is_flag=True, default=False)
 @click.pass_obj
 def count(obj, by_type):
     def calculate_host_number(config, type='all', servers=None):
         hosts = set()
         if type == 'all':
             for values in config['ENNSTATUS_SERVERS'].values():
                 ips = frozenset(values['IPS'])
                 hosts.add(ips)
             return len(hosts)
         else:
             for server in servers[servertype]:
                 ips = frozenset(
                     config['ENNSTATUS_SERVERS'][server.name.lower()]['IPS']
+                )
                 hosts.add(ips)
         return len(hosts)
     app = create_app()
     with app.app_context():
         app.logger.disabled = True
         servers = split_all_servers_to_types()
     if not by_type:
         click.echo(
             'We have %s servers in total!' % (
                 click.style(
                     str(sum(len(x) for x in servers.values())),
                     fg='blue'
+                )
+            )
+        )
         click.echo(
             'We have %s different hosts!' % (
                 click.style(
                     str(calculate_host_number(app.config)),
                     fg='blue'
+                )
+            )
+        )
     else:
         for servertype, server in servers.items():
             click.echo(
                 'We have %s %s servers!' % (
                     click.style(
                         str(len(server)),
                         fg='blue'
                     ),
                     click.style(
                         servertype,
                         fg='red'
+                    )
+                )
+            )
             click.echo(
                 'We have %s different %s hosts!' % (
                     click.style(
                         str(
                             calculate_host_number(
                                 app.config,
                                 type=servertype,
                                 servers=servers
+                            )
                         ),
                         fg='blue'
                     ),
                     click.style(
                         servertype,
                         fg='red'
+                    )
+                )
+            )
 @stats.command('countries')
 @click.option('--by-type', 'by_type', is_flag=True, default=False)
 @click.pass_obj
 def countries(obj, by_type):
     app = create_app()
     with app.app_context():
         app.logger.disabled = True
         servers = split_all_servers_to_types()
     if not by_type:
         countries = defaultdict(int)
         for key, value in servers.items():
             for server in value:
                 countries[server.country] += 1
         for key, value in sorted(countries.items(), key=lambda x: x[1]):
             click.echo(
                 '%s: %s' % (
                     click.style(key, fg='green'),
                     click.style(str(value), fg='blue')
+                )
+            )
         click.echo(
             'We are hosted in %s different countries' % click.style(
                 str(len(countries.keys())),
                 fg='blue'
+            )
+        )
     else:
         type_countries = {
             'exit': defaultdict(int),
             'relay': defaultdict(int),
             'bridge': defaultdict(int)
+        }
         for key, value in servers.items():
             for server in value:
                 type_countries[key][server.country] += 1
         type_countries = dict((k, v) for k, v in type_countries.items() if v)
         for key, value in sorted(type_countries.items(), key=lambda x: x[0]):
             click.echo(
                 click.style(
                     key.capitalize(),
                     fg='red',
                     bold=True,
                     underline=True
+                )
+            )
             for country, count in sorted(type_countries[key].items(), key=lambda x: x[1]):
                 click.echo(
                     '%s: %s' % (
                         click.style(country, fg='green'),
                         click.style(str(count), fg='blue')
+                    )
+                )
             click.echo(
                 '%s are hosted in %s different countries' % (
                     key,
                     click.style(
                         str(len(type_countries[key].keys())),
                         fg='blue'
+                    )
+                )
+            )
 @stats.command('exit_probability')
 @click.option('--by-server', 'by_server', is_flag=True, default=False)
 @click.pass_obj
 def exit_probability(obj, by_server):
     app = create_app()
     with app.app_context():
         app.logger.disabled = True
         servers = split_all_servers_to_types()
     if not by_server:
         exit_probability = defaultdict(int)
         for server in servers['exit']:
             for subkey in ('1_week', '1_month', '3_months', '1_year', '5_years'):
                 if server.mean_exit_probability[subkey] is not None:
                     exit_probability[subkey] += server.mean_exit_probability[subkey]
         for subkey in ('1_week', '1_month', '3_months', '1_year', '5_years'):
             click.echo(
                 'Mean exit probability over %s: %s' % (
                     click.style(
                         subkey,
                         fg='blue'
                     ),
                     click.style(
                         str(round(exit_probability[subkey], 2)) + '%',
                         fg='red'
+                    )
+                )
+            )
     else:
         for server in servers['exit']:
             click.echo(
                 click.style(
                     server.name.capitalize(),
                     fg='red',
                     bold=True,
                     underline=True
+                )
+            )
             for subkey in ('1_week', '1_month', '3_months', '1_year', '5_years'):
                 if server.mean_exit_probability[subkey] is not None:
                     click.echo(
                         'Mean exit probabilty over %s: %s' % (
                             click.style(
                                 subkey,
                                 fg='blue'
                             ),
                             click.style(
                                 str(round(server.mean_exit_probability[subkey], 2)) + "%",
                                 fg='red'
+                            )
+                        )
+                    )

requirements.in

➞

Show inline comments

 Babel
 click
 Flask-Bootstrap
 Flask-HTTPAuth
 Flask-Mail
 Flask-Moment
 Flask-WTF
 Flask
 jsonschema
 pygeoip
 python-gnupg
 requests
 strict-rfc3339
 OnionPy

requirements.txt

➞

Show inline comments

@@ @@ -9,20 +9,21 @@ blinker==1.4 # via flask-ma @@
 click==6.3
 dominate==2.1.17          # via flask-bootstrap
 Flask-Bootstrap==3.3.5.7
 Flask-HTTPAuth==2.7.2
 Flask-Mail==0.9.1
 Flask-Moment==0.5.1
 Flask-WTF==0.12
 Flask==0.10.1             # via flask-bootstrap, flask-httpauth, flask-mail, flask-moment, flask-wtf
 itsdangerous==0.24        # via flask
 Jinja2==2.8               # via flask
 jsonschema==2.5.1
 MarkupSafe==0.23          # via jinja2
 onionpy==0.3.2
 pygeoip==0.3.2
 python-gnupg==0.3.8
 pytz==2015.7              # via babel
 requests==2.9.1
+requests==2.9.1           # via onionpy
 strict-rfc3339==0.6
 visitor==0.1.2            # via flask-bootstrap
 Werkzeug==0.11.4          # via flask, flask-wtf
 WTForms==2.1              # via flask-wtf

setup.py

➞

Show inline comments

@@ @@ -2,25 +2,25 @@ from setuptools import setup, find_packa @@
 def _get_requirements():
     with open('requirements.in', encoding='utf-8') as f:
         lines = f.readlines()
     lines = [line[:-1] for line in lines if not line.startswith('#')]
     return lines
 setup(name='Ennstatus',
-      version='5.4.4',
+      version='5.4.5',
       description=('Ennstatus provides the user with vital information about '
                    'the status of the organizations Tor servers.'),
       author='Frënn vun der Ënn',
       author_email='info@enn.lu',
       url='https://bitbucket.org/fvde/ennstatus',
       license='GPLv3+',
       packages=find_packages(),
       install_requires=_get_requirements(),
       include_package_data=True,
       entry_points={
           'console_scripts': [
               'ennstatuscli = ennstatus.cli:cli',

0 comments (0 inline, 0 general)