diff --git a/onionr/api.py b/onionr/api.py index adaa3cd7..3bd6214d 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -17,61 +17,20 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import random, threading, hmac, base64, time, os, json, socket -from gevent.pywsgi import WSGIServer, WSGIHandler -from gevent import Timeout +import threading, hmac, base64, time, os +from gevent.pywsgi import WSGIServer import flask from flask import request, Response, abort, send_from_directory import core -import onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config, onionrblockapi +import onionrexceptions, onionrcrypto, logger, config import httpapi from httpapi import friendsapi, profilesapi, configapi, miscpublicapi, miscclientapi, insertblock, onionrsitesapi from onionrservices import httpheaders import onionr from onionrutils import bytesconverter, stringvalidators, epoch, mnemonickeys -from httpapi import apiutils +from httpapi import apiutils, security, fdsafehandler config.reload() -class FDSafeHandler(WSGIHandler): - '''Our WSGI handler. Doesn't do much non-default except timeouts''' - def handle(self): - self.timeout = Timeout(120, Exception) - self.timeout.start() - try: - WSGIHandler.handle(self) - except Exception: - self.handle_error() - finally: - self.timeout.close() - - def handle_error(self): - if v is self.timeout: - self.result = [b"Timeout"] - self.start_response("200 OK", []) - self.process_result() - else: - WSGIHandler.handle_error(self) - -def setBindIP(filePath=''): - '''Set a random localhost IP to a specified file (intended for private or public API localhost IPs)''' - if config.get('general.random_bind_ip', True): - hostOctets = [str(127), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF))] - data = '.'.join(hostOctets) - # Try to bind IP. Some platforms like Mac block non normal 127.x.x.x - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - try: - s.bind((data, 0)) - except OSError: - # if mac/non-bindable, show warning and default to 127.0.0.1 - logger.warn('Your platform appears to not support random local host addresses 127.x.x.x. Falling back to 127.0.0.1.') - data = '127.0.0.1' - s.close() - else: - data = '127.0.0.1' - if filePath != '': - with open(filePath, 'w') as bindFile: - bindFile.write(data) - return data class PublicAPI: ''' @@ -82,94 +41,27 @@ class PublicAPI: app = flask.Flask('PublicAPI') self.i2pEnabled = config.get('i2p.host', False) self.hideBlocks = [] # Blocks to be denied sharing - self.host = setBindIP(clientAPI._core.publicApiHostFile) + self.host = apiutils.setbindip.set_bind_IP(clientAPI._core.publicApiHostFile, clientAPI._core) self.torAdder = clientAPI._core.hsAddress self.i2pAdder = clientAPI._core.i2pAddress self.bindPort = config.get('client.public.port') self.lastRequest = 0 self.hitCount = 0 # total rec requests to public api since server started + self.config = config + self.clientAPI = clientAPI + self.API_VERSION = onionr.API_VERSION logger.info('Running public api on %s:%s' % (self.host, self.bindPort)) - @app.before_request - def validateRequest(): - '''Validate request has the correct hostname''' - # If high security level, deny requests to public (HS should be disabled anyway for Tor, but might not be for I2P) - if config.get('general.security_level', default=1) > 0: - abort(403) - if type(self.torAdder) is None and type(self.i2pAdder) is None: - # abort if our hs addresses are not known - abort(403) - if request.host not in (self.i2pAdder, self.torAdder): - # Disallow connection if wrong HTTP hostname, in order to prevent DNS rebinding attacks - abort(403) - self.hitCount += 1 # raise hit count for valid requests - - @app.after_request - def sendHeaders(resp): - '''Send api, access control headers''' - resp = httpheaders.set_default_onionr_http_headers(resp) - # Network API version - resp.headers['X-API'] = onionr.API_VERSION - self.lastRequest = epoch.get_rounded_epoch(roundS=5) - return resp - - @app.route('/') - def banner(): - # Display a bit of information to people who visit a node address in their browser - try: - with open('static-data/index.html', 'r') as html: - resp = Response(html.read(), mimetype='text/html') - except FileNotFoundError: - resp = Response("") - return resp - - @app.route('/getblocklist') - def getBlockList(): - return httpapi.miscpublicapi.public_block_list(clientAPI, self, request) - - @app.route('/getdata/') - def getBlockData(name): - # Share data for a block if we have it - return httpapi.miscpublicapi.public_get_block_data(clientAPI, self, name) - - @app.route('/www/') - def wwwPublic(path): - # A way to share files directly over your .onion - if not config.get("www.public.run", True): - abort(403) - return send_from_directory(config.get('www.public.path', 'static-data/www/public/'), path) - - @app.route('/ping') - def ping(): - # Endpoint to test if nodes are up - return Response("pong!") - - @app.route('/pex') - def peerExchange(): - response = ','.join(clientAPI._core.listAdders(recent=3600)) - if len(response) == 0: - response = '' - return Response(response) - - @app.route('/announce', methods=['post']) - def acceptAnnounce(): - resp = httpapi.miscpublicapi.announce(clientAPI, request) - return resp - - @app.route('/upload', methods=['post']) - def upload(): - '''Accept file uploads. In the future this will be done more often than on creation - to speed up block sync - ''' - return httpapi.miscpublicapi.upload(clientAPI, request) - # Set instances, then startup our public api server clientAPI.setPublicAPIInstance(self) while self.torAdder == '': clientAPI._core.refreshFirstStartVars() self.torAdder = clientAPI._core.hsAddress time.sleep(0.1) - self.httpServer = WSGIServer((self.host, self.bindPort), app, log=None, handler_class=FDSafeHandler) + + app.register_blueprint(security.public.PublicAPISecurity(self).public_api_security_bp) + app.register_blueprint(miscpublicapi.endpoints.PublicEndpoints(self).public_endpoints_bp) + self.httpServer = WSGIServer((self.host, self.bindPort), app, log=None, handler_class=fdsafehandler.FDSafeHandler) self.httpServer.serve_forever() class API: @@ -200,12 +92,13 @@ class API: self.publicAPI = None # gets set when the thread calls our setter... bad hack but kinda necessary with flask #threading.Thread(target=PublicAPI, args=(self,)).start() - self.host = setBindIP(self._core.privateApiHostFile) + self.host = apiutils.setbindip.set_bind_IP(self._core.privateApiHostFile, self._core) logger.info('Running api on %s:%s' % (self.host, self.bindPort)) self.httpServer = '' self.queueResponse = {} onionrInst.setClientAPIInst(self) + app.register_blueprint(security.client.ClientAPISecurity(self).client_api_security_bp) app.register_blueprint(friendsapi.friends) app.register_blueprint(profilesapi.profile_BP) app.register_blueprint(configapi.config_BP) @@ -303,7 +196,7 @@ class API: def getHumanReadable(name): return Response(mnemonickeys.get_human_readable_ID(name)) - self.httpServer = WSGIServer((self.host, bindPort), app, log=None, handler_class=FDSafeHandler) + self.httpServer = WSGIServer((self.host, bindPort), app, log=None, handler_class=fdsafehandler.FDSafeHandler) self.httpServer.serve_forever() def setPublicAPIInstance(self, inst): diff --git a/onionr/communicatorutils/lookupadders.py b/onionr/communicatorutils/lookupadders.py index c0394e3d..fc4527dc 100755 --- a/onionr/communicatorutils/lookupadders.py +++ b/onionr/communicatorutils/lookupadders.py @@ -21,7 +21,7 @@ import logger from onionrutils import stringvalidators def lookup_new_peer_transports_with_communicator(comm_inst): - logger.info('Looking up new addresses...', terminal=True) + logger.info('Looking up new addresses...') tryAmount = 1 newPeers = [] for i in range(tryAmount): diff --git a/onionr/httpapi/apiutils/__init__.py b/onionr/httpapi/apiutils/__init__.py index 29b0c778..cab13c98 100644 --- a/onionr/httpapi/apiutils/__init__.py +++ b/onionr/httpapi/apiutils/__init__.py @@ -1,44 +1,3 @@ -import json -import core, onionrblockapi -from onionrutils import bytesconverter, stringvalidators -from . import shutdown +from . import shutdown, setbindip, getblockdata -class GetBlockData: - def __init__(self, client_api_inst=None): - if client_api_inst is None: - self.client_api_inst = None - self.c = core.Core() - elif isinstance(client_api_inst, core.Core): - self.client_api_inst = None - self.c = client_api_inst - else: - self.client_api_Inst = client_api_inst - self.c = core.Core() - - def get_block_data(self, bHash, decrypt=False, raw=False, headerOnly=False): - assert stringvalidators.validate_hash(bHash) - bl = onionrblockapi.Block(bHash, core=self.c) - if decrypt: - bl.decrypt() - if bl.isEncrypted and not bl.decrypted: - raise ValueError - - if not raw: - if not headerOnly: - retData = {'meta':bl.bheader, 'metadata': bl.bmetadata, 'content': bl.bcontent} - for x in list(retData.keys()): - try: - retData[x] = retData[x].decode() - except AttributeError: - pass - else: - validSig = False - signer = bytesconverter.bytes_to_str(bl.signer) - if bl.isSigned() and stringvalidators.validate_pub_key(signer) and bl.isSigner(signer): - validSig = True - bl.bheader['validSig'] = validSig - bl.bheader['meta'] = '' - retData = {'meta': bl.bheader, 'metadata': bl.bmetadata} - return json.dumps(retData) - else: - return bl.raw \ No newline at end of file +GetBlockData = getblockdata.GetBlockData \ No newline at end of file diff --git a/onionr/httpapi/apiutils/getblockdata.py b/onionr/httpapi/apiutils/getblockdata.py new file mode 100644 index 00000000..881c0c89 --- /dev/null +++ b/onionr/httpapi/apiutils/getblockdata.py @@ -0,0 +1,42 @@ +import json +import core, onionrblockapi +from onionrutils import bytesconverter, stringvalidators +class GetBlockData: + def __init__(self, client_api_inst=None): + if client_api_inst is None: + self.client_api_inst = None + self.c = core.Core() + elif isinstance(client_api_inst, core.Core): + self.client_api_inst = None + self.c = client_api_inst + else: + self.client_api_Inst = client_api_inst + self.c = core.Core() + + def get_block_data(self, bHash, decrypt=False, raw=False, headerOnly=False): + assert stringvalidators.validate_hash(bHash) + bl = onionrblockapi.Block(bHash, core=self.c) + if decrypt: + bl.decrypt() + if bl.isEncrypted and not bl.decrypted: + raise ValueError + + if not raw: + if not headerOnly: + retData = {'meta':bl.bheader, 'metadata': bl.bmetadata, 'content': bl.bcontent} + for x in list(retData.keys()): + try: + retData[x] = retData[x].decode() + except AttributeError: + pass + else: + validSig = False + signer = bytesconverter.bytes_to_str(bl.signer) + if bl.isSigned() and stringvalidators.validate_pub_key(signer) and bl.isSigner(signer): + validSig = True + bl.bheader['validSig'] = validSig + bl.bheader['meta'] = '' + retData = {'meta': bl.bheader, 'metadata': bl.bmetadata} + return json.dumps(retData) + else: + return bl.raw \ No newline at end of file diff --git a/onionr/httpapi/apiutils/setbindip.py b/onionr/httpapi/apiutils/setbindip.py new file mode 100644 index 00000000..6a8e512b --- /dev/null +++ b/onionr/httpapi/apiutils/setbindip.py @@ -0,0 +1,25 @@ +import random, socket +import config, logger +def set_bind_IP(filePath='', core_inst=None): + '''Set a random localhost IP to a specified file (intended for private or public API localhost IPs)''' + if not core_inst is None: + config = core_inst.config + + if config.get('general.random_bind_ip', True): + hostOctets = [str(127), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF))] + data = '.'.join(hostOctets) + # Try to bind IP. Some platforms like Mac block non normal 127.x.x.x + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + s.bind((data, 0)) + except OSError: + # if mac/non-bindable, show warning and default to 127.0.0.1 + logger.warn('Your platform appears to not support random local host addresses 127.x.x.x. Falling back to 127.0.0.1.') + data = '127.0.0.1' + s.close() + else: + data = '127.0.0.1' + if filePath != '': + with open(filePath, 'w') as bindFile: + bindFile.write(data) + return data \ No newline at end of file diff --git a/onionr/httpapi/fdsafehandler.py b/onionr/httpapi/fdsafehandler.py new file mode 100644 index 00000000..ef45565e --- /dev/null +++ b/onionr/httpapi/fdsafehandler.py @@ -0,0 +1,21 @@ +from gevent.pywsgi import WSGIServer, WSGIHandler +from gevent import Timeout +class FDSafeHandler(WSGIHandler): + '''Our WSGI handler. Doesn't do much non-default except timeouts''' + def handle(self): + self.timeout = Timeout(120, Exception) + self.timeout.start() + try: + WSGIHandler.handle(self) + except Exception: + self.handle_error() + finally: + self.timeout.close() + + def handle_error(self): + if v is self.timeout: + self.result = [b"Timeout"] + self.start_response("200 OK", []) + self.process_result() + else: + WSGIHandler.handle_error(self) \ No newline at end of file diff --git a/onionr/httpapi/miscpublicapi/__init__.py b/onionr/httpapi/miscpublicapi/__init__.py index b29454f7..2f5f9c56 100755 --- a/onionr/httpapi/miscpublicapi/__init__.py +++ b/onionr/httpapi/miscpublicapi/__init__.py @@ -1,23 +1,4 @@ -''' - Onionr - Private P2P Communication - - Public endpoints to do various block sync actions and announcement -''' -''' - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -''' -from . import announce, upload, getblocks +from . import announce, upload, getblocks, endpoints announce = announce.handle_announce # endpoint handler for accepting peer announcements upload = upload.accept_upload # endpoint handler for accepting public uploads diff --git a/onionr/httpapi/miscpublicapi/endpoints.py b/onionr/httpapi/miscpublicapi/endpoints.py new file mode 100644 index 00000000..269e63c9 --- /dev/null +++ b/onionr/httpapi/miscpublicapi/endpoints.py @@ -0,0 +1,77 @@ +''' + Onionr - Private P2P Communication + + Misc public API endpoints too small to need their own file and that need access to the public api inst +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +from flask import Response, Blueprint, request, send_from_directory, abort +from . import getblocks, upload, announce +class PublicEndpoints: + def __init__(self, public_api): + client_API = public_api.clientAPI + config = client_API._core.config + + public_endpoints_bp = Blueprint('publicendpoints', __name__) + self.public_endpoints_bp = public_endpoints_bp + @public_endpoints_bp.route('/') + def banner(): + # Display a bit of information to people who visit a node address in their browser + try: + with open('static-data/index.html', 'r') as html: + resp = Response(html.read(), mimetype='text/html') + except FileNotFoundError: + resp = Response("") + return resp + + @public_endpoints_bp.route('/getblocklist') + def get_block_list(): + return getblocks.public_block_list(client_API, public_api, request) + + @public_endpoints_bp.route('/getdata/') + def get_block_data(name): + # Share data for a block if we have it + return getblocks.public_get_block_data(client_API, public_api, name) + + @public_endpoints_bp.route('/www/') + def www_public(path): + # A way to share files directly over your .onion + if not config.get("www.public.run", True): + abort(403) + return send_from_directory(config.get('www.public.path', 'static-data/www/public/'), path) + + @public_endpoints_bp.route('/ping') + def ping(): + # Endpoint to test if nodes are up + return Response("pong!") + + @public_endpoints_bp.route('/pex') + def peer_exchange(): + response = ','.join(client_API._core.listAdders(recent=3600)) + if len(response) == 0: + response = '' + return Response(response) + + @public_endpoints_bp.route('/announce', methods=['post']) + def accept_announce(): + resp = httpapi.miscpublicapi.announce(client_API, request) + return resp + + @public_endpoints_bp.route('/upload', methods=['post']) + def upload(): + '''Accept file uploads. In the future this will be done more often than on creation + to speed up block sync + ''' + return upload.accept_upload(client_API, request) \ No newline at end of file diff --git a/onionr/httpapi/security/client.py b/onionr/httpapi/security/client.py index 65798ad7..4191e67f 100644 --- a/onionr/httpapi/security/client.py +++ b/onionr/httpapi/security/client.py @@ -1,6 +1,25 @@ +''' + Onionr - Private P2P Communication + + Process incoming requests to the client api server to validate they are legitimate +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' import hmac from flask import Blueprint, request, abort - +from onionrservices import httpheaders # Be extremely mindful of this. These are endpoints available without a password whitelist_endpoints = ('siteapi.site', 'www', 'staticfiles.onionrhome', 'staticfiles.homedata', 'staticfiles.board', 'staticfiles.profiles', @@ -9,28 +28,34 @@ whitelist_endpoints = ('siteapi.site', 'www', 'staticfiles.onionrhome', 'staticf 'staticfiles.mail', 'staticfiles.mailindex', 'staticfiles.friends', 'staticfiles.friendsindex', 'staticfiles.clandestine', 'staticfiles.clandestineIndex') -@app.before_request -def validateRequest(): - '''Validate request has set password and is the correct hostname''' - # For the purpose of preventing DNS rebinding attacks - if request.host != '%s:%s' % (self.host, self.bindPort): - abort(403) - if request.endpoint in whitelist_endpoints: - return - try: - if not hmac.compare_digest(request.headers['token'], self.clientToken): - if not hmac.compare_digest(request.form['token'], self.clientToken): - abort(403) - except KeyError: - if not hmac.compare_digest(request.form['token'], self.clientToken): - abort(403) +class ClientAPISecurity: + def __init__(self, client_api): + client_api_security_bp = Blueprint('clientapisecurity', __name__) + self.client_api_security_bp = client_api_security_bp + self.client_api = client_api -@app.after_request -def afterReq(resp): - # Security headers - resp = httpheaders.set_default_onionr_http_headers(resp) - if request.endpoint == 'site': - resp.headers['Content-Security-Policy'] = "default-src 'none'; style-src data: 'unsafe-inline'; img-src data:" - else: - resp.headers['Content-Security-Policy'] = "default-src 'none'; script-src 'self'; object-src 'none'; style-src 'self'; img-src 'self'; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'self'" - return resp \ No newline at end of file + @client_api_security_bp.before_app_request + def validate_request(): + '''Validate request has set password and is the correct hostname''' + # For the purpose of preventing DNS rebinding attacks + if request.host != '%s:%s' % (client_api.host, client_api.bindPort): + abort(403) + if request.endpoint in whitelist_endpoints: + return + try: + if not hmac.compare_digest(request.headers['token'], client_api.clientToken): + if not hmac.compare_digest(request.form['token'], client_api.clientToken): + abort(403) + except KeyError: + if not hmac.compare_digest(request.form['token'], client_api.clientToken): + abort(403) + + @client_api_security_bp.after_app_request + def after_req(resp): + # Security headers + resp = httpheaders.set_default_onionr_http_headers(resp) + if request.endpoint == 'site': + resp.headers['Content-Security-Policy'] = "default-src 'none'; style-src data: 'unsafe-inline'; img-src data:" + else: + resp.headers['Content-Security-Policy'] = "default-src 'none'; script-src 'self'; object-src 'none'; style-src 'self'; img-src 'self'; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'self'" + return resp \ No newline at end of file diff --git a/onionr/httpapi/security/public.py b/onionr/httpapi/security/public.py index e69de29b..5e580792 100644 --- a/onionr/httpapi/security/public.py +++ b/onionr/httpapi/security/public.py @@ -0,0 +1,50 @@ +''' + Onionr - Private P2P Communication + + Process incoming requests to the public api server for certain attacks +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +from flask import Blueprint, request, abort +from onionrservices import httpheaders +from onionrutils import epoch + +class PublicAPISecurity: + def __init__(self, public_api): + public_api_security_bp = Blueprint('publicapisecurity', __name__) + self.public_api_security_bp = public_api_security_bp + + @public_api_security_bp.before_app_request + def validate_request(): + '''Validate request has the correct hostname''' + # If high security level, deny requests to public (HS should be disabled anyway for Tor, but might not be for I2P) + if public_api.config.get('general.security_level', default=1) > 0: + abort(403) + if type(public_api.torAdder) is None and type(public_api.i2pAdder) is None: + # abort if our hs addresses are not known + abort(403) + if request.host not in (public_api.i2pAdder, public_api.torAdder): + # Disallow connection if wrong HTTP hostname, in order to prevent DNS rebinding attacks + abort(403) + public_api.hitCount += 1 # raise hit count for valid requests + + @public_api_security_bp.after_app_request + def send_headers(resp): + '''Send api, access control headers''' + resp = httpheaders.set_default_onionr_http_headers(resp) + # Network API version + resp.headers['X-API'] = public_api.API_VERSION + public_api.lastRequest = epoch.get_rounded_epoch(roundS=5) + return resp \ No newline at end of file diff --git a/onionr/onionrservices/connectionserver.py b/onionr/onionrservices/connectionserver.py index 413aa6ca..e97b30e0 100755 --- a/onionr/onionrservices/connectionserver.py +++ b/onionr/onionrservices/connectionserver.py @@ -23,7 +23,7 @@ from flask import Flask import core, logger, httpapi import onionrexceptions from netcontroller import getOpenPort -import api +from httpapi import apiutils from onionrutils import stringvalidators, basicrequests, bytesconverter from . import httpheaders @@ -40,7 +40,7 @@ class ConnectionServer: socks = core_inst.config.get('tor.socksport') # Load config for Tor socks port for proxy service_app = Flask(__name__) # Setup Flask app for server. service_port = getOpenPort() - service_ip = api.setBindIP() + service_ip = apiutils.setbindip.set_bind_IP(core_inst=self.core_inst) http_server = WSGIServer(('127.0.0.1', service_port), service_app, log=None) core_inst.onionrInst.communicatorInst.service_greenlets.append(http_server)