From 14f2d03ebffc6b6aa70a12deffee9f53ea84aa1f Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Wed, 26 Aug 2020 08:25:43 +0000 Subject: [PATCH] Formatting improvements in httpapis --- src/httpapi/fdsafehandler.py | 4 ++- src/httpapi/insertblock.py | 4 +-- src/httpapi/miscpublicapi/announce.py | 1 - src/httpapi/miscpublicapi/endpoints.py | 32 +++++++++++---------- src/httpapi/miscpublicapi/getblocks.py | 9 ++++-- src/httpapi/miscpublicapi/upload.py | 3 +- src/httpapi/security/client.py | 37 +++++++++++++++++-------- src/httpapi/security/lan.py | 3 +- src/httpapi/security/pluginwhitelist.py | 2 +- src/httpapi/security/public.py | 10 ++++--- src/lan/__init__.py | 1 - src/lan/client/__init__.py | 6 ++-- src/lan/discover.py | 14 ++++------ src/lan/server/__init__.py | 22 ++++++++++----- 14 files changed, 88 insertions(+), 60 deletions(-) diff --git a/src/httpapi/fdsafehandler.py b/src/httpapi/fdsafehandler.py index e132911e..a9a9aed2 100644 --- a/src/httpapi/fdsafehandler.py +++ b/src/httpapi/fdsafehandler.py @@ -1,5 +1,7 @@ from gevent.pywsgi import WSGIServer, WSGIHandler from gevent import Timeout + + class FDSafeHandler(WSGIHandler): '''Our WSGI handler. Doesn't do much non-default except timeouts''' def handle(self): @@ -10,5 +12,5 @@ class FDSafeHandler(WSGIHandler): except Timeout as ex: if ex is self.timeout: pass - else: + else: raise diff --git a/src/httpapi/insertblock.py b/src/httpapi/insertblock.py index 693cee74..bafe41e1 100644 --- a/src/httpapi/insertblock.py +++ b/src/httpapi/insertblock.py @@ -10,7 +10,7 @@ from flask import Blueprint, Response, request, g if TYPE_CHECKING: from deadsimplekv import DeadSimpleKV - + import onionrblocks from onionrcrypto import hashers from onionrutils import bytesconverter @@ -78,7 +78,7 @@ def client_api_insert_block(): pass try: - # The setting in the UI is for if forward secrecy is enabled, not disabled + # Setting in the mail UI is for if forward secrecy is *enabled* disable_forward_secrecy = not insert_data['forward'] except KeyError: disable_forward_secrecy = False diff --git a/src/httpapi/miscpublicapi/announce.py b/src/httpapi/miscpublicapi/announce.py index ac2dbba6..c3ba80a7 100755 --- a/src/httpapi/miscpublicapi/announce.py +++ b/src/httpapi/miscpublicapi/announce.py @@ -9,7 +9,6 @@ import logger from etc import onionrvalues from onionrutils import stringvalidators, bytesconverter import filepaths -from communicator import OnionrCommunicatorDaemon """ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/httpapi/miscpublicapi/endpoints.py b/src/httpapi/miscpublicapi/endpoints.py index 6c5d1da7..f8db4b06 100644 --- a/src/httpapi/miscpublicapi/endpoints.py +++ b/src/httpapi/miscpublicapi/endpoints.py @@ -1,9 +1,13 @@ -''' - Onionr - Private P2P Communication +"""Onionr - Private P2P Communication. - Misc public API endpoints too small to need their own file and that need access to the public api inst -''' -''' +Misc public API endpoints too small to need their own file +and that need access to the public api inst +""" +from flask import Response, Blueprint, request, send_from_directory, abort, g +from . import getblocks, upload, announce +from coredb import keydb +import config +""" This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or @@ -16,11 +20,9 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -''' -from flask import Response, Blueprint, request, send_from_directory, abort, g -from . import getblocks, upload, announce -from coredb import keydb -import config +""" + + class PublicEndpoints: def __init__(self, public_api): @@ -39,7 +41,8 @@ class PublicEndpoints: @public_endpoints_bp.route('/getblocklist') def get_block_list(): - '''Get a list of blocks, optionally filtered by epoch time stamp, excluding those hidden''' + """Get a list of blocks, optionally filtered by epoch time stamp, + excluding those hidden""" return getblocks.get_public_block_list(public_api, request) @public_endpoints_bp.route('/getdata/') @@ -68,14 +71,15 @@ class PublicEndpoints: @public_endpoints_bp.route('/announce', methods=['post']) def accept_announce(): - '''Accept announcements with pow token to prevent spam''' + """Accept announcements with pow token to prevent spam""" g.shared_state = public_api._too_many resp = announce.handle_announce(request) return resp @public_endpoints_bp.route('/upload', methods=['post']) def upload_endpoint(): - '''Accept file uploads. In the future this will be done more often than on creation + """Accept file uploads. + In the future this will be done more often than on creation to speed up block sync - ''' + """ return upload.accept_upload(request) \ No newline at end of file diff --git a/src/httpapi/miscpublicapi/getblocks.py b/src/httpapi/miscpublicapi/getblocks.py index de405b52..10c2786a 100755 --- a/src/httpapi/miscpublicapi/getblocks.py +++ b/src/httpapi/miscpublicapi/getblocks.py @@ -45,11 +45,14 @@ def get_block_data(publicAPI, data): """data is the block hash in hex""" resp = '' if stringvalidators.validate_hash(data): - if not config.get('general.hide_created_blocks', True) or data not in publicAPI.hideBlocks: + if not config.get('general.hide_created_blocks', True) \ + or data not in publicAPI.hideBlocks: if data in publicAPI._too_many.get(BlockList).get(): - block = apiutils.GetBlockData().get_block_data(data, raw=True, decrypt=False) + block = apiutils.GetBlockData().get_block_data( + data, raw=True, decrypt=False) try: - block = block.encode('utf-8') # Encode in case data is binary + # Encode in case data is binary + block = block.encode('utf-8') except AttributeError: if len(block) == 0: abort(404) diff --git a/src/httpapi/miscpublicapi/upload.py b/src/httpapi/miscpublicapi/upload.py index 8c268726..699f3094 100755 --- a/src/httpapi/miscpublicapi/upload.py +++ b/src/httpapi/miscpublicapi/upload.py @@ -5,7 +5,6 @@ Accept block uploads to the public API server import sys from gevent import spawn -from gevent import threading from flask import Response from flask import abort from flask import g @@ -43,7 +42,7 @@ def accept_upload(request): if g.too_many.get_by_string("DeadSimpleKV").get('onlinePeers'): spawn( localcommand.local_command, - f'/daemon-event/upload_event', + '/daemon-event/upload_event', post=True, is_json=True, post_data={'block': b_hash} diff --git a/src/httpapi/security/client.py b/src/httpapi/security/client.py index 77cb5d06..b8c04ca4 100644 --- a/src/httpapi/security/client.py +++ b/src/httpapi/security/client.py @@ -1,6 +1,7 @@ """Onionr - Private P2P Communication. -Process incoming requests to the client api server to validate they are legitimate +Process incoming requests to the client api server to validate +that they are legitimate and not DNSR/XSRF or other local adversary """ import hmac from flask import Blueprint, request, abort, g @@ -21,17 +22,23 @@ from . import pluginwhitelist along with this program. If not, see . """ -# Be extremely mindful of this. These are endpoints available without a password -whitelist_endpoints = ['www', 'staticfiles.homedata', 'staticfiles.sharedContent', -'staticfiles.friends', 'staticfiles.friendsindex', 'siteapi.site', 'siteapi.siteFile', 'staticfiles.onionrhome', -'themes.getTheme', 'staticfiles.onboarding', 'staticfiles.onboardingIndex'] +# Be extremely mindful of this. +# These are endpoints available without a password +whitelist_endpoints = [ + 'www', 'staticfiles.homedata', + 'staticfiles.sharedContent', + 'staticfiles.friends', 'staticfiles.friendsindex', 'siteapi.site', + 'siteapi.siteFile', 'staticfiles.onionrhome', + 'themes.getTheme', 'staticfiles.onboarding', 'staticfiles.onboardingIndex'] + class ClientAPISecurity: def __init__(self, client_api): client_api_security_bp = Blueprint('clientapisecurity', __name__) self.client_api_security_bp = client_api_security_bp self.client_api = client_api - pluginwhitelist.load_plugin_security_whitelist_endpoints(whitelist_endpoints) + pluginwhitelist.load_plugin_security_whitelist_endpoints( + whitelist_endpoints) @client_api_security_bp.before_app_request def validate_request(): @@ -48,14 +55,18 @@ class ClientAPISecurity: if request.endpoint in whitelist_endpoints: return - if request.path.startswith('/site/'): return + if request.path.startswith('/site/'): + return try: - if not hmac.compare_digest(request.headers['token'], client_api.clientToken): - if not hmac.compare_digest(request.form['token'], client_api.clientToken): + if not hmac.compare_digest( + request.headers['token'], client_api.clientToken): + if not hmac.compare_digest( + request.form['token'], client_api.clientToken): abort(403) except KeyError: - if not hmac.compare_digest(request.form['token'], client_api.clientToken): + if not hmac.compare_digest( + request.form['token'], client_api.clientToken): abort(403) @client_api_security_bp.after_app_request @@ -63,7 +74,9 @@ class ClientAPISecurity: # Security headers resp = httpheaders.set_default_onionr_http_headers(resp) if request.endpoint in ('siteapi.site', 'siteapi.siteFile'): - resp.headers['Content-Security-Policy'] = "default-src 'none'; style-src 'self' data: 'unsafe-inline'; img-src 'self' data:; media-src 'self' data:" + resp.headers['Content-Security-Policy'] = \ + "default-src 'none'; style-src 'self' data: 'unsafe-inline'; img-src 'self' data:; media-src 'self' data:" # noqa else: - resp.headers['Content-Security-Policy'] = "default-src 'none'; script-src 'self'; object-src 'none'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; media-src 'self'; frame-src 'none'; font-src 'self'; connect-src 'self'" + resp.headers['Content-Security-Policy'] = \ + "default-src 'none'; script-src 'self'; object-src 'none'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; media-src 'self'; frame-src 'none'; font-src 'self'; connect-src 'self'" # noqa return resp diff --git a/src/httpapi/security/lan.py b/src/httpapi/security/lan.py index 1989188c..baacfa00 100644 --- a/src/httpapi/security/lan.py +++ b/src/httpapi/security/lan.py @@ -59,7 +59,8 @@ class LANAPISecurity: 'Clear-Site-Data', 'Referrer-Policy') try: if g.is_onionr_client: - for header in NON_NETWORK_HEADERS: del resp.headers[header] + for header in NON_NETWORK_HEADERS: + del resp.headers[header] except AttributeError: abort(403) lan_client.lastRequest = epoch.get_rounded_epoch(roundS=5) diff --git a/src/httpapi/security/pluginwhitelist.py b/src/httpapi/security/pluginwhitelist.py index ecfa280f..e508a8fa 100644 --- a/src/httpapi/security/pluginwhitelist.py +++ b/src/httpapi/security/pluginwhitelist.py @@ -30,4 +30,4 @@ def load_plugin_security_whitelist_endpoints(whitelist: list): try: whitelist.extend(getattr(plugin, "security_whitelist")) except AttributeError: - pass \ No newline at end of file + pass diff --git a/src/httpapi/security/public.py b/src/httpapi/security/public.py index aa63bba7..f38a5278 100644 --- a/src/httpapi/security/public.py +++ b/src/httpapi/security/public.py @@ -64,13 +64,15 @@ class PublicAPISecurity: # Network API version resp.headers['X-API'] = public_api.API_VERSION # Delete some HTTP headers for Onionr user agents - NON_NETWORK_HEADERS = ('Content-Security-Policy', 'X-Frame-Options', - 'X-Content-Type-Options', 'Feature-Policy', - 'Clear-Site-Data', 'Referrer-Policy') + NON_NETWORK_HEADERS = ( + 'Content-Security-Policy', 'X-Frame-Options', + 'X-Content-Type-Options', 'Feature-Policy', + 'Clear-Site-Data', 'Referrer-Policy') try: if g.is_onionr_client: - for header in NON_NETWORK_HEADERS: del resp.headers[header] + for header in NON_NETWORK_HEADERS: + del resp.headers[header] except AttributeError: abort(403) diff --git a/src/lan/__init__.py b/src/lan/__init__.py index f900a889..ea30efd5 100644 --- a/src/lan/__init__.py +++ b/src/lan/__init__.py @@ -31,7 +31,6 @@ class LANManager: self.too_many = too_many self.peers: "exploded IP Address string" = [] - def start(self): Thread(target=learn_services, daemon=True).start() Thread(target=advertise_service, daemon=True).start() diff --git a/src/lan/client/__init__.py b/src/lan/client/__init__.py index aa0a53da..de57c2dd 100644 --- a/src/lan/client/__init__.py +++ b/src/lan/client/__init__.py @@ -7,7 +7,6 @@ import requests from typing import Set from onionrtypes import LANIP -from utils.bettersleep import better_sleep import logger from coredb.blockmetadb import get_block_list from onionrblocks.blockimporter import import_block_from_data @@ -38,8 +37,9 @@ def _lan_work(peer: LANIP): blocks = requests.get(url + 'blist/0').text.splitlines() for block in blocks: if block not in our_blocks: - import_block_from_data(requests.get(url + f'get/{block}', stream=True).raw.read(6000000)) - + import_block_from_data( + requests.get( + url + f'get/{block}', stream=True).raw.read(6000000)) for port in ports: try: diff --git a/src/lan/discover.py b/src/lan/discover.py index 52ac1b46..aae3e1f8 100644 --- a/src/lan/discover.py +++ b/src/lan/discover.py @@ -4,10 +4,7 @@ Discover and publish private-network """ import socket import struct -from typing import TYPE_CHECKING -from typing import List from ipaddress import ip_address -from socket import SHUT_RDWR from .getip import lan_ips, best_ip from utils.bettersleep import better_sleep @@ -32,7 +29,6 @@ IS_ALL_GROUPS = True ANNOUNCE_LOOP_SLEEP = 30 - def learn_services(): """Take a list to infintely add lan service info to.""" @@ -54,12 +50,13 @@ def learn_services(): continue service_ips = service_ips.replace('onionr-', '').split('-') - port = 0 for service in service_ips: try: ip_address(service) - if not ip_address(service).is_private: raise ValueError - if service in lan_ips: raise ValueError + if not ip_address(service).is_private: + raise ValueError + if service in lan_ips: + raise ValueError except ValueError: pass else: @@ -70,7 +67,8 @@ def advertise_service(specific_ips=None): # regarding socket.IP_MULTICAST_TTL # --------------------------------- # for all packets sent, after three hops on the network the packet will not - # be re-sent/broadcast (see https://www.tldp.org/HOWTO/Multicast-HOWTO-6.html) + # be re-sent/broadcast + # (see https://www.tldp.org/HOWTO/Multicast-HOWTO-6.html) MULTICAST_TTL = 3 ips = best_ip diff --git a/src/lan/server/__init__.py b/src/lan/server/__init__.py index 72abd10b..ce52a8c2 100644 --- a/src/lan/server/__init__.py +++ b/src/lan/server/__init__.py @@ -39,6 +39,7 @@ from utils.bettersleep import better_sleep ports = range(1337, 1340) _start_time = time.time() + class LANServer: def __init__(self, shared_state): app = Flask(__name__) @@ -51,12 +52,14 @@ class LANServer: @app.before_request def dns_rebinding_prevention(): - if request.remote_addr in lan_ips or ipaddress.ip_address(request.remote_addr).is_loopback: + if request.remote_addr in lan_ips or \ + ipaddress.ip_address(request.remote_addr).is_loopback: if time.time() - _start_time > 600: abort(403) if request.host != f'{self.host}:{self.port}': logger.warn('Potential DNS rebinding attack on LAN server:') - logger.warn(f'Hostname {request.host} was used instead of {self.host}:{self.port}') + logger.warn( + f'Hostname {request.host} was used instead of {self.host}:{self.port}') # noqa abort(403) @app.route('/blist/