Changeset - 47d07b5cb8db
[Not reviewed]
Dennis Fink - 9 years ago 2016-03-02 20:01:22
dennis.fink@c3l.lu
Added flags
3 files changed with 20 insertions and 0 deletions:
0 comments (0 inline, 0 general)
ennstatus/api/model.py
Show inline comments
 
@@ -27,232 +27,242 @@ import strict_rfc3339
 

	
 
from flask import current_app
 
from pkg_resources import resource_filename
 
from onion_py.manager import Manager
 
from onion_py.caching import OnionSimpleCache
 

	
 
from ..utils import check_ip
 

	
 

	
 
schema = json.load(
 
    open(
 
        resource_filename('ennstatus.api', 'schema/server.json'),
 
        encoding='utf-8'
 
    )
 
)
 

	
 
validate = functools.partial(
 
    jsonschema.validate,
 
    schema=schema,
 
    format_checker=jsonschema.FormatChecker()
 
)
 

	
 
manager = Manager(OnionSimpleCache())
 

	
 

	
 
def calculate_weight(data):
 

	
 
    obj = {}
 

	
 
    for subkey in ('1_week', '1_month', '3_months', '1_year', '5_years'):
 

	
 
        try:
 
            subdata = data[subkey]
 
        except KeyError:
 
            continue
 

	
 
        factor = subdata.factor
 

	
 
        values = [x * factor for x in subdata.values if x is not None]
 

	
 
        if values:
 
            obj[subkey] = statistics.mean(values) * 100
 
        else:
 
            obj[subkey] = None
 

	
 
    return obj
 

	
 

	
 
class ServerEncoder(json.JSONEncoder):
 

	
 
    def default(self, obj):
 

	
 
        if isinstance(obj, (ipaddress.IPv4Address, ipaddress.IPv6Address)):
 
            return str(obj)
 

	
 
        if isinstance(obj, datetime):
 
            return strict_rfc3339.timestamp_to_rfc3339_utcoffset(
 
                obj.timestamp()
 
            )
 

	
 
        return json.JSONEncoder.default(self, obj)
 

	
 

	
 
class ServerDecoder(json.JSONDecoder):
 

	
 
    def decode(self, json_string):
 

	
 
        default_obj = super().decode(json_string)
 

	
 
        for key in ('ip', 'ip6'):
 
            if key in default_obj:
 
                current_app.logger.debug('{}: {}'.format(
 
                    key, default_obj[key]
 
                )
 
                )
 
                default_obj[key] = ipaddress.ip_address(default_obj[key])
 

	
 
        current_app.logger.debug('Loading last_updated')
 
        default_obj['last_updated'] = datetime.fromtimestamp(
 
            strict_rfc3339.rfc3339_to_timestamp(default_obj['last_updated'])
 
        )
 

	
 
        return default_obj
 

	
 

	
 
class Server:
 

	
 
    def __init__(self, *args, **kwargs):
 

	
 
        self.name = kwargs['name']
 
        self.type = kwargs['type']
 
        self.status = kwargs.get('status')
 
        self.fingerprint = kwargs['fingerprint']
 
        self.last_updated = kwargs['last_updated']
 
        self.country = kwargs['country']
 
        self.bandwidth = kwargs.get('bandwidth')
 
        self.flags = kwargs.get('flags')
 

	
 
        if self.type == 'bridge':
 
            self.obfs = kwargs.get('obfs')
 
            self.fteproxy = kwargs.get('fteproxy')
 
            self.flashproxy = kwargs.get('flashproxy')
 
            self.meek = kwargs.get('meek')
 
        else:
 
            self.ip = kwargs['ip']
 

	
 
            if 'ip6' in kwargs:
 
                self.ip6 = kwargs['ip6']
 

	
 
            default_weights = {
 
                '1_week': None,
 
                '1_month': None,
 
                '3_months': None,
 
                '1_year': None,
 
                '5_years': None
 
            }
 

	
 
            self.mean_consensus_weight = kwargs.get(
 
                'mean_consensus_weight',
 
                default_weights
 
            )
 
            self.mean_guard_probability = kwargs.get(
 
                'mean_guard_probability',
 
                default_weights
 
            )
 
            self.mean_exit_probability = kwargs.get(
 
                'mean_exit_probability',
 
                default_weights
 
            )
 
            self.mean_consensus_weight_fraction = kwargs.get(
 
                'mean_consensus_weight_fraction',
 
                default_weights
 
            )
 
            self.mean_middle_probability = kwargs.get(
 
                'mean_middle_probability',
 
                default_weights
 
            )
 

	
 
    @classmethod
 
    def from_file_by_name(cls, name):
 

	
 
        filepath = Path('data') / (name.lower() + '.json')
 

	
 
        current_app.logger.info('Loading {}'.format(str(filepath)))
 
        if filepath.exists() and filepath.is_file():
 
            try:
 
                with filepath.open(encoding='utf-8') as f:
 
                    data = json.load(f, cls=ServerDecoder)
 
            except (IOError, ValueError):
 
                current_app.logger.error('IOError or ValueError')
 
                return False
 
            else:
 
                return cls(**data)
 
        else:
 
            current_app.logger.error('File error!')
 
            return False
 

	
 
    @classmethod
 
    def from_json(cls, server):
 

	
 
        try:
 
            if cls.check_json_format(json.loads(server)):
 
                decoded = json.loads(server, cls=ServerDecoder)
 
                return cls(**decoded)
 
        except (jsonschema.ValidationError, ValueError) as e:
 
            raise e
 

	
 
    @classmethod
 
    def from_dict(cls, server):
 
        return cls.from_json(json.dumps(server))
 

	
 
    def json(self):
 
        return json.dumps(self.__dict__, cls=ServerEncoder)
 

	
 
    @staticmethod
 
    def check_json_format(server):
 

	
 
        try:
 
            validate(server)
 
        except jsonschema.ValidationError as e:
 
            raise e
 

	
 
        for key in ('ip', 'ip6'):
 
            if key in server:
 
                address = ipaddress.ip_address(server[key])
 
                if not check_ip(address):
 
                    raise ValueError('{} is not accepted!\n'.format(key))
 
        return True
 

	
 
    def save(self):
 

	
 
        filepath = Path('data') / (self.name.lower() + '.json')
 

	
 
        try:
 
            with filepath.open(mode='w', encoding='utf-8') as f:
 
                json.dump(self.__dict__, f, cls=ServerEncoder)
 
        except Exception as e:
 
            raise e
 

	
 
    def update_weights(self):
 

	
 
        if self.type not in ('exit', 'relay'):
 
            raise NotImplementedError
 

	
 
        try:
 
            data = manager.query('weights', lookup=self.fingerprint)
 
        except:
 
            raise NotImplementedError
 

	
 
        if data is not None:
 
            data = data.relays[0]
 

	
 
        self.mean_consensus_weight = calculate_weight(data.consensus_weight)
 
        self.mean_exit_probability = calculate_weight(data.exit_probability)
 
        self.mean_guard_probability = calculate_weight(
 
            data.guard_probability
 
        )
 
        self.mean_middle_probability = calculate_weight(
 
            data.middle_probability
 
        )
 
        self.mean_consensus_weight_fraction = calculate_weight(
 
            data.consensus_weight_fraction
 
        )
 

	
 
    def update_flags(self):
 

	
 
        try:
 
            data = manager.query('details', lookup=self.fingerprint)
 
        except:
 
            raise NotImplementedError
 

	
 
        self.flags = data.relays[0].flags
 

	
 
    def check_status(self):
 

	
 
        now = datetime.utcnow()
 
        delta = now - self.last_updated
 

	
 
        if delta.seconds >= 3600:
 
            self.status = False
 
        elif delta.seconds >= 600:
 
            self.status = None
ennstatus/api/schema/server.json
Show inline comments
 
@@ -7,192 +7,197 @@
 
            "type": "string"
 
        },
 
        "status": {
 
            "$ref": "#/definitions/status_data"
 
        },
 
        "country": {
 
            "type": "string",
 
            "enum": [
 
                "Afghanistan", "Aland Islands", "Albania", "Algeria", 
 
                "American Samoa", "Andorra", "Angola", "Anguilla",
 
                "Anonymous Proxy", "Antarctica", "Antigua and Barbuda",
 
                "Argentina", "Armenia", "Aruba", "Asia/Pacific Region",
 
                "Australia", "Austria", "Azerbaijan", "Bahamas", "Bahrain",
 
                "Bangladesh", "Barbados", "Belarus", "Belgium", "Belize",
 
                "Benin", "Bermuda", "Bhutan", "Bolivia",
 
                "Bonaire, Sint Eustatius and Saba", "Bosnia and Herzegovina",
 
                "Botswana", "Bouvet Island", "Brazil",
 
                "British Indian Ocean Territory", "Brunei Darussalam",
 
                "Bulgaria", "Burkina Faso", "Burundi", "Cambodia",
 
                "Cameroon", "Canada", "Cape Verde", "Cayman Islands",
 
                "Central African Republic", "Chad", "Chile", "China",
 
                "Christmas Island", "Cocos (Keeling) Islands", "Colombia",
 
                "Comoros", "Congo", "Congo, The Democratic Republic of the",
 
                "Cook Islands", "Costa Rica", "Cote D'Ivoire", "Croatia",
 
                "Cuba", "Cyprus", "Czech Republic", "Denmark", "Djibouti",
 
                "Dominica", "Dominican Republic", "Ecuador", "Egypt",
 
                "El Salvador", "Equatorial Guinea", "Eritrea", "Estonia",
 
                "Ethiopia", "Europe", "Falkland Islands (Malvinas)",
 
                "Faroe Islands", "Fiji", "Finland", "France",
 
                "France, Metropolitan", "French Guiana", "French Polynesia",
 
                "French Southern Territories", "Gabon", "Gambia", "Georgia",
 
                "Germany", "Ghana", "Gibraltar", "Greece", "Greenland",
 
                "Grenada", "Guadeloupe", "Guam", "Guatemala", "Guernsey",
 
                "Guinea", "Guinea-Bissau", "Guyana", "Haiti",
 
                "Heard Island and McDonald Islands",
 
                "Holy See (Vatican City State)", "Honduras", "Hong Kong",
 
                "Hungary", "Iceland", "India", "Indonesia",
 
                "Iran, Islamic Republic of", "Iraq", "Ireland", "Isle of Man",
 
                "Israel", "Italy", "Jamaica", "Japan", "Jersey", "Jordan",
 
                "Kazakhstan", "Kenya", "Kiribati",
 
                "Korea, Democratic People's Republic of",
 
                "Korea, Republic of", "Kuwait", "Kyrgyzstan",
 
                "Lao People's Democratic Republic", "Latvia", "Lebanon",
 
                "Lesotho", "Liberia", "Libya", "Liechtenstein", "Lithuania",
 
                "Luxembourg", "Macau", "Macedonia", "Madagascar", "Malawi",
 
                "Malaysia", "Maldives", "Mali", "Malta", "Marshall Islands",
 
                "Martinique", "Mauritania", "Mauritius", "Mayotte", "Mexico",
 
                "Micronesia, Federated States of", "Moldova, Republic of",
 
                "Monaco", "Mongolia", "Montenegro", "Montserrat", "Morocco",
 
                "Mozambique", "Myanmar", "Namibia", "Nauru", "Nepal",
 
                "Netherlands", "Netherlands Antilles", "New Caledonia",
 
                "New Zealand", "Nicaragua", "Niger", "Nigeria", "Niue",
 
                "Norfolk Island", "Northern Mariana Islands", "Norway",
 
                "Oman", "Other", "Pakistan", "Palau", "Palestinian Territory",
 
                "Panama", "Papua New Guinea", "Paraguay", "Peru",
 
                "Philippines", "Pitcairn Islands", "Poland", "Portugal",
 
                "Puerto Rico", "Qatar", "Reunion", "Romania",
 
                "Russian Federation", "Rwanda", "Saint Barthelemy",
 
                "Saint Helena", "Saint Kitts and Nevis", "Saint Lucia",
 
                "Saint Martin", "Saint Pierre and Miquelon",
 
                "Saint Vincent and the Grenadines", "Samoa", "San Marino",
 
                "Sao Tome and Principe", "Satellite Provider", "Saudi Arabia",
 
                "Senegal", "Serbia", "Seychelles", "Sierra Leone", "Singapore",
 
                "Slovakia", "Slovenia", "Solomon Islands", "Somalia",
 
                "South Africa", "South Georgia and the South Sandwich Islands",
 
                "South Sudan", "Spain", "Sri Lanka", "Sudan", "Suriname",
 
                "Svalbard and Jan Mayen", "Swaziland", "Sweden", "Switzerland",
 
                "Syrian Arab Republic", "Taiwan", "Tajikistan",
 
                "Tanzania, United Republic of", "Thailand", "Timor-Leste",
 
                "Togo", "Tokelau", "Tonga", "Trinidad and Tobago", "Tunisia",
 
                "Turkey", "Turkmenistan", "Turks and Caicos Islands", "Tuvalu",
 
                "Uganda", "Ukraine", "United Arab Emirates", "United Kingdom",
 
                "United States", "United States Minor Outlying Islands",
 
                "Uruguay", "Uzbekistan", "Vanuatu", "Venezuela", "Vietnam",
 
                "Virgin Islands, British", "Virgin Islands, U.S.",
 
                "Wallis and Futuna", "Western Sahara", "Yemen", "Zambia",
 
                "Zimbabwe"
 
            ]
 
        },
 
        "fingerprint": {
 
            "type": "string",
 
            "pattern": "^[a-zA-Z0-9]{40}$"
 
        },
 
        "last_updated": {
 
            "type": "string",
 
            "format": "date-time"
 
        },
 
        "bandwidth": {
 
            "type": [
 
                "string",
 
                "null"
 
            ],
 
            "default": null
 
        },
 
        "type": {
 
            "type": "string"
 
        },
 
        "flags": {
 
            "type": "array",
 
            "items": { "type": "string" },
 
            "uniqueItems": true
 
        }
 
    },
 
    "required": [
 
        "name",
 
        "status",
 
        "last_updated",
 
        "country",
 
        "fingerprint",
 
        "type"
 
    ],
 
    "definitions": {
 
        "weights": {
 
            "type": "object",
 
            "properties": {
 
                "1_week": {
 
                    "$ref": "#/definitions/weights_data"
 
                },
 
                "1_month": {
 
                    "$ref": "#/definitions/weights_data"
 
                },
 
                "3_months": {
 
                    "$ref": "#/definitions/weights_data"
 
                },
 
                "1_year": {
 
                    "$ref": "#/definitions/weights_data"
 
                },
 
                "5_years": {
 
                    "$ref": "#/definitions/weights_data"
 
                }
 
            },
 
            "required": [
 
                "1_week",
 
                "1_month",
 
                "3_months",
 
                "1_year",
 
                "5_years"
 
            ],
 
            "additionalProperties": false
 
        },
 
        "weights_data": {
 
            "type": [
 
                "number",
 
                "null"
 
            ],
 
            "default": null
 
        },
 
        "status_data": {
 
            "type": [
 
                "boolean",
 
                "null"
 
            ],
 
            "default": null
 
        }
 
    },
 
    "anyOf": [
 
        {
 
            "properties": {
 
                "type": {
 
                    "enum": [
 
                        "exit",
 
                        "relay"
 
                    ]
 
                },
 
                "ip": {
 
                    "type": "string",
 
                    "format": "ipv4"
 
                },
 
                "ip6": {
 
                    "type": "string",
 
                    "format": "ipv6"
 
                },
 
                "mean_consensus_weight": {
 
                    "$ref": "#/definitions/weights"
 
                },
 
                "mean_guard_probability": {
 
                    "$ref": "#/definitions/weights"
 
                },
 
                "mean_exit_probability": {
 
                    "$ref": "#/definitions/weights"
 
                },
 
                "mean_middle_probability": {
 
                    "$ref": "#/definitions/weights"
 
                },
 
                "mean_consensus_weight_fraction": {
 
                    "$ref": "#/definitions/weights"
 
                }
 
            },
 
            "required": [
 
                "ip"
 
            ],
 
            "not": {
 
                "required": [
 
                    "obfs",
 
                    "fteproxy",
 
                    "flashproxy",
 
                    "meek"
ennstatus/api/views.py
Show inline comments
 
@@ -28,163 +28,168 @@ import strict_rfc3339
 
import pygeoip
 

	
 
from ennstatus import csrf
 
from ennstatus.status.functions import (single_server, all_servers,
 
                                        all_servers_by_type)
 
from .model import Server
 
from .auth import httpauth
 

	
 
api_page = Blueprint('api', __name__)
 
gi4 = pygeoip.GeoIP('/usr/share/GeoIP/GeoIP.dat', pygeoip.MEMORY_CACHE)
 
gi6 = pygeoip.GeoIP('/usr/share/GeoIP/GeoIPv6.dat', pygeoip.MEMORY_CACHE)
 

	
 

	
 
@csrf.exempt
 
@api_page.route('/update', methods=('POST',))
 
@httpauth.login_required
 
def update():
 

	
 
    current_app.logger.info('Handling update')
 

	
 
    try:
 
        servers = current_app.config['ENNSTATUS_SERVERS']
 
    except KeyError as e:
 
        current_app.logger.error(str(e))
 
        return abort(500)
 

	
 
    username = httpauth.username()
 

	
 
    try:
 
        if request.remote_addr not in servers[username]['IPS']:
 
            current_app.logger.warn(
 
                'Unallowed IP {} tried to update data!'.format(
 
                    request.remote_addr
 
                )
 
            )
 
            return 'IP not allowed!\n', 403, {'Content-Type': 'text/plain'}
 
    except KeyError as e:
 
        current_app.logger.error(str(e))
 
        return abort(500)
 

	
 
    try:
 
        data = request.get_json()
 
    except BadRequest:
 
        current_app.logger.info('No JSON data supplied!')
 
        return 'No JSON data supplied!\n', 400, {'Content-Type': 'text/plain'}
 

	
 
    try:
 
        if username != data['name'].lower():
 
            current_app.logger.warn(
 
                'Unallowed user {} tried to update {}!'.format(
 
                    username, data['name']
 
                )
 
            )
 
            return ('You are not allowed to update this server\n',
 
                    403, {'Content-Type': 'text/plain'})
 
    except KeyError:
 
        return abort(409)
 

	
 
    if 'ip' in data:
 
        ip = data['ip']
 
    elif 'ip6' in data:
 
        ip = data['ip6']
 
    else:
 
        ip = request.remote_addr
 
        try:
 
            temp_ip = ipaddress.ip_address(ip)
 
        except ipaddress.AddressValueError:
 
            return 'IP not allowed!\n', 403, {'Content-Type': 'text/plain'}
 
        else:
 
            if temp_ip.version == 4:
 
                data['ip'] = ip
 
            elif temp_ip.verison == 6:
 
                data['ip6'] = ip
 

	
 
    try:
 
        country = gi4.country_name_by_addr(ip)
 
    except pygeoip.GeoIPError:
 
        country = gi6.country_name_by_addr(ip)
 

	
 
    data['country'] = country
 
    data['last_updated'] = strict_rfc3339.timestamp_to_rfc3339_utcoffset(
 
        datetime.utcnow().timestamp()
 
    )
 

	
 
    try:
 
        server = Server.from_dict(data)
 
    except Exception as e:
 
        current_app.logger.warning(' '.join([str(e), str(data)]))
 
        return str(e), 409, {'Content-Type': 'text/plain'}
 

	
 
    try:
 
        server.update_weights()
 
    except NotImplementedError:
 
        pass
 

	
 
    try:
 
        server.update_flags()
 
    except NotImplementedError:
 
        pass
 

	
 
    try:
 
        server.save()
 
    except Exception as e:
 
        current_app.logger.error(str(e))
 
        return str(e), 500, {'Content-Type': 'text/plain'}
 

	
 
    current_app.logger.info('Return result')
 
    return (
 
        server.json(), 201,
 
        {
 
            'Location': '/api/export/json/single?server_name={}'.format(
 
                server.name
 
            )
 
        }
 
    )
 

	
 

	
 
@api_page.route('/export', defaults={'server_type': 'all'})
 
@api_page.route(('/export/<any("all", "exit", "bridge", "relay", "single")'
 
                 ':server_type>'))
 
def export(server_type):
 

	
 
    current_app.logger.info('Handling export')
 
    if server_type == 'single':
 
        server_name = request.args.get('server_name', None)
 
        if server_name is not None:
 
            server = single_server(server_name)
 
            if server:
 
                return server.json(), 200, {'Content-Type': 'application/json'}
 
            else:
 
                current_app.logger.warning('Server not found!')
 
                return ('Server not found!\n',
 
                        404, {'Content-Type': 'text/plain'})
 
        else:
 
            current_app.logger.warning('No server_name specified!')
 
            return ('No server_name specified!\n',
 
                    400, {'Content-Type': 'text/plain'})
 

	
 
    else:
 
        if server_type == 'all':
 
            current_app.logger.info('Getting all servers!')
 
            servers = [
 
                json.loads(
 
                    server.json()
 
                ) for server in all_servers()
 
            ]
 
        else:
 
            current_app.logger.info('Getting all {}!'.format(server_type))
 
            servers = [
 
                json.loads(
 
                    server.json()
 
                ) for server in all_servers_by_type(server_type.lower())
 
            ]
 

	
 
        return json.dumps(servers), 200, {'Content-Type': 'application/json'}
 

	
 

	
 
@api_page.route('/fingerprints', defaults={'server_type': 'all'})
 
@api_page.route('/fingerprints/<any("all", "exit", "relay"):server_type>')
 
def fingerprint(server_type):
 
    if server_type == 'all':
 
        servers = [server.fingerprint for server in all_servers()
 
                   if server.type != 'bridge']
 
    else:
 
        servers = [server.fingerprint
 
                   for server in all_servers_by_type(server_type.lower())]
 

	
 
    return '\n'.join(servers), 200, {'Content-Type': 'text/plain'}
0 comments (0 inline, 0 general)