refactored api servers, especially the public server. Client server to recieve similar refactoring soon

This commit is contained in:
Kevin Froman 2019-07-02 01:32:26 -05:00
parent 54dabefe45
commit 10ef9ce41b
11 changed files with 286 additions and 213 deletions

View File

@ -17,61 +17,20 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import random, threading, hmac, base64, time, os, json, socket
from gevent.pywsgi import WSGIServer, WSGIHandler
from gevent import Timeout
import threading, hmac, base64, time, os
from gevent.pywsgi import WSGIServer
import flask
from flask import request, Response, abort, send_from_directory
import core
import onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config, onionrblockapi
import onionrexceptions, onionrcrypto, logger, config
import httpapi
from httpapi import friendsapi, profilesapi, configapi, miscpublicapi, miscclientapi, insertblock, onionrsitesapi
from onionrservices import httpheaders
import onionr
from onionrutils import bytesconverter, stringvalidators, epoch, mnemonickeys
from httpapi import apiutils
from httpapi import apiutils, security, fdsafehandler
config.reload()
class FDSafeHandler(WSGIHandler):
'''Our WSGI handler. Doesn't do much non-default except timeouts'''
def handle(self):
self.timeout = Timeout(120, Exception)
self.timeout.start()
try:
WSGIHandler.handle(self)
except Exception:
self.handle_error()
finally:
self.timeout.close()
def handle_error(self):
if v is self.timeout:
self.result = [b"Timeout"]
self.start_response("200 OK", [])
self.process_result()
else:
WSGIHandler.handle_error(self)
def setBindIP(filePath=''):
'''Set a random localhost IP to a specified file (intended for private or public API localhost IPs)'''
if config.get('general.random_bind_ip', True):
hostOctets = [str(127), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF))]
data = '.'.join(hostOctets)
# Try to bind IP. Some platforms like Mac block non normal 127.x.x.x
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.bind((data, 0))
except OSError:
# if mac/non-bindable, show warning and default to 127.0.0.1
logger.warn('Your platform appears to not support random local host addresses 127.x.x.x. Falling back to 127.0.0.1.')
data = '127.0.0.1'
s.close()
else:
data = '127.0.0.1'
if filePath != '':
with open(filePath, 'w') as bindFile:
bindFile.write(data)
return data
class PublicAPI:
'''
@ -82,94 +41,27 @@ class PublicAPI:
app = flask.Flask('PublicAPI')
self.i2pEnabled = config.get('i2p.host', False)
self.hideBlocks = [] # Blocks to be denied sharing
self.host = setBindIP(clientAPI._core.publicApiHostFile)
self.host = apiutils.setbindip.set_bind_IP(clientAPI._core.publicApiHostFile, clientAPI._core)
self.torAdder = clientAPI._core.hsAddress
self.i2pAdder = clientAPI._core.i2pAddress
self.bindPort = config.get('client.public.port')
self.lastRequest = 0
self.hitCount = 0 # total rec requests to public api since server started
self.config = config
self.clientAPI = clientAPI
self.API_VERSION = onionr.API_VERSION
logger.info('Running public api on %s:%s' % (self.host, self.bindPort))
@app.before_request
def validateRequest():
'''Validate request has the correct hostname'''
# If high security level, deny requests to public (HS should be disabled anyway for Tor, but might not be for I2P)
if config.get('general.security_level', default=1) > 0:
abort(403)
if type(self.torAdder) is None and type(self.i2pAdder) is None:
# abort if our hs addresses are not known
abort(403)
if request.host not in (self.i2pAdder, self.torAdder):
# Disallow connection if wrong HTTP hostname, in order to prevent DNS rebinding attacks
abort(403)
self.hitCount += 1 # raise hit count for valid requests
@app.after_request
def sendHeaders(resp):
'''Send api, access control headers'''
resp = httpheaders.set_default_onionr_http_headers(resp)
# Network API version
resp.headers['X-API'] = onionr.API_VERSION
self.lastRequest = epoch.get_rounded_epoch(roundS=5)
return resp
@app.route('/')
def banner():
# Display a bit of information to people who visit a node address in their browser
try:
with open('static-data/index.html', 'r') as html:
resp = Response(html.read(), mimetype='text/html')
except FileNotFoundError:
resp = Response("")
return resp
@app.route('/getblocklist')
def getBlockList():
return httpapi.miscpublicapi.public_block_list(clientAPI, self, request)
@app.route('/getdata/<name>')
def getBlockData(name):
# Share data for a block if we have it
return httpapi.miscpublicapi.public_get_block_data(clientAPI, self, name)
@app.route('/www/<path:path>')
def wwwPublic(path):
# A way to share files directly over your .onion
if not config.get("www.public.run", True):
abort(403)
return send_from_directory(config.get('www.public.path', 'static-data/www/public/'), path)
@app.route('/ping')
def ping():
# Endpoint to test if nodes are up
return Response("pong!")
@app.route('/pex')
def peerExchange():
response = ','.join(clientAPI._core.listAdders(recent=3600))
if len(response) == 0:
response = ''
return Response(response)
@app.route('/announce', methods=['post'])
def acceptAnnounce():
resp = httpapi.miscpublicapi.announce(clientAPI, request)
return resp
@app.route('/upload', methods=['post'])
def upload():
'''Accept file uploads. In the future this will be done more often than on creation
to speed up block sync
'''
return httpapi.miscpublicapi.upload(clientAPI, request)
# Set instances, then startup our public api server
clientAPI.setPublicAPIInstance(self)
while self.torAdder == '':
clientAPI._core.refreshFirstStartVars()
self.torAdder = clientAPI._core.hsAddress
time.sleep(0.1)
self.httpServer = WSGIServer((self.host, self.bindPort), app, log=None, handler_class=FDSafeHandler)
app.register_blueprint(security.public.PublicAPISecurity(self).public_api_security_bp)
app.register_blueprint(miscpublicapi.endpoints.PublicEndpoints(self).public_endpoints_bp)
self.httpServer = WSGIServer((self.host, self.bindPort), app, log=None, handler_class=fdsafehandler.FDSafeHandler)
self.httpServer.serve_forever()
class API:
@ -200,12 +92,13 @@ class API:
self.publicAPI = None # gets set when the thread calls our setter... bad hack but kinda necessary with flask
#threading.Thread(target=PublicAPI, args=(self,)).start()
self.host = setBindIP(self._core.privateApiHostFile)
self.host = apiutils.setbindip.set_bind_IP(self._core.privateApiHostFile, self._core)
logger.info('Running api on %s:%s' % (self.host, self.bindPort))
self.httpServer = ''
self.queueResponse = {}
onionrInst.setClientAPIInst(self)
app.register_blueprint(security.client.ClientAPISecurity(self).client_api_security_bp)
app.register_blueprint(friendsapi.friends)
app.register_blueprint(profilesapi.profile_BP)
app.register_blueprint(configapi.config_BP)
@ -303,7 +196,7 @@ class API:
def getHumanReadable(name):
return Response(mnemonickeys.get_human_readable_ID(name))
self.httpServer = WSGIServer((self.host, bindPort), app, log=None, handler_class=FDSafeHandler)
self.httpServer = WSGIServer((self.host, bindPort), app, log=None, handler_class=fdsafehandler.FDSafeHandler)
self.httpServer.serve_forever()
def setPublicAPIInstance(self, inst):

View File

@ -21,7 +21,7 @@ import logger
from onionrutils import stringvalidators
def lookup_new_peer_transports_with_communicator(comm_inst):
logger.info('Looking up new addresses...', terminal=True)
logger.info('Looking up new addresses...')
tryAmount = 1
newPeers = []
for i in range(tryAmount):

View File

@ -1,44 +1,3 @@
import json
import core, onionrblockapi
from onionrutils import bytesconverter, stringvalidators
from . import shutdown
from . import shutdown, setbindip, getblockdata
class GetBlockData:
def __init__(self, client_api_inst=None):
if client_api_inst is None:
self.client_api_inst = None
self.c = core.Core()
elif isinstance(client_api_inst, core.Core):
self.client_api_inst = None
self.c = client_api_inst
else:
self.client_api_Inst = client_api_inst
self.c = core.Core()
def get_block_data(self, bHash, decrypt=False, raw=False, headerOnly=False):
assert stringvalidators.validate_hash(bHash)
bl = onionrblockapi.Block(bHash, core=self.c)
if decrypt:
bl.decrypt()
if bl.isEncrypted and not bl.decrypted:
raise ValueError
if not raw:
if not headerOnly:
retData = {'meta':bl.bheader, 'metadata': bl.bmetadata, 'content': bl.bcontent}
for x in list(retData.keys()):
try:
retData[x] = retData[x].decode()
except AttributeError:
pass
else:
validSig = False
signer = bytesconverter.bytes_to_str(bl.signer)
if bl.isSigned() and stringvalidators.validate_pub_key(signer) and bl.isSigner(signer):
validSig = True
bl.bheader['validSig'] = validSig
bl.bheader['meta'] = ''
retData = {'meta': bl.bheader, 'metadata': bl.bmetadata}
return json.dumps(retData)
else:
return bl.raw
GetBlockData = getblockdata.GetBlockData

View File

@ -0,0 +1,42 @@
import json
import core, onionrblockapi
from onionrutils import bytesconverter, stringvalidators
class GetBlockData:
def __init__(self, client_api_inst=None):
if client_api_inst is None:
self.client_api_inst = None
self.c = core.Core()
elif isinstance(client_api_inst, core.Core):
self.client_api_inst = None
self.c = client_api_inst
else:
self.client_api_Inst = client_api_inst
self.c = core.Core()
def get_block_data(self, bHash, decrypt=False, raw=False, headerOnly=False):
assert stringvalidators.validate_hash(bHash)
bl = onionrblockapi.Block(bHash, core=self.c)
if decrypt:
bl.decrypt()
if bl.isEncrypted and not bl.decrypted:
raise ValueError
if not raw:
if not headerOnly:
retData = {'meta':bl.bheader, 'metadata': bl.bmetadata, 'content': bl.bcontent}
for x in list(retData.keys()):
try:
retData[x] = retData[x].decode()
except AttributeError:
pass
else:
validSig = False
signer = bytesconverter.bytes_to_str(bl.signer)
if bl.isSigned() and stringvalidators.validate_pub_key(signer) and bl.isSigner(signer):
validSig = True
bl.bheader['validSig'] = validSig
bl.bheader['meta'] = ''
retData = {'meta': bl.bheader, 'metadata': bl.bmetadata}
return json.dumps(retData)
else:
return bl.raw

View File

@ -0,0 +1,25 @@
import random, socket
import config, logger
def set_bind_IP(filePath='', core_inst=None):
'''Set a random localhost IP to a specified file (intended for private or public API localhost IPs)'''
if not core_inst is None:
config = core_inst.config
if config.get('general.random_bind_ip', True):
hostOctets = [str(127), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF))]
data = '.'.join(hostOctets)
# Try to bind IP. Some platforms like Mac block non normal 127.x.x.x
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.bind((data, 0))
except OSError:
# if mac/non-bindable, show warning and default to 127.0.0.1
logger.warn('Your platform appears to not support random local host addresses 127.x.x.x. Falling back to 127.0.0.1.')
data = '127.0.0.1'
s.close()
else:
data = '127.0.0.1'
if filePath != '':
with open(filePath, 'w') as bindFile:
bindFile.write(data)
return data

View File

@ -0,0 +1,21 @@
from gevent.pywsgi import WSGIServer, WSGIHandler
from gevent import Timeout
class FDSafeHandler(WSGIHandler):
'''Our WSGI handler. Doesn't do much non-default except timeouts'''
def handle(self):
self.timeout = Timeout(120, Exception)
self.timeout.start()
try:
WSGIHandler.handle(self)
except Exception:
self.handle_error()
finally:
self.timeout.close()
def handle_error(self):
if v is self.timeout:
self.result = [b"Timeout"]
self.start_response("200 OK", [])
self.process_result()
else:
WSGIHandler.handle_error(self)

View File

@ -1,23 +1,4 @@
'''
Onionr - Private P2P Communication
Public endpoints to do various block sync actions and announcement
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
from . import announce, upload, getblocks
from . import announce, upload, getblocks, endpoints
announce = announce.handle_announce # endpoint handler for accepting peer announcements
upload = upload.accept_upload # endpoint handler for accepting public uploads

View File

@ -0,0 +1,77 @@
'''
Onionr - Private P2P Communication
Misc public API endpoints too small to need their own file and that need access to the public api inst
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
from flask import Response, Blueprint, request, send_from_directory, abort
from . import getblocks, upload, announce
class PublicEndpoints:
def __init__(self, public_api):
client_API = public_api.clientAPI
config = client_API._core.config
public_endpoints_bp = Blueprint('publicendpoints', __name__)
self.public_endpoints_bp = public_endpoints_bp
@public_endpoints_bp.route('/')
def banner():
# Display a bit of information to people who visit a node address in their browser
try:
with open('static-data/index.html', 'r') as html:
resp = Response(html.read(), mimetype='text/html')
except FileNotFoundError:
resp = Response("")
return resp
@public_endpoints_bp.route('/getblocklist')
def get_block_list():
return getblocks.public_block_list(client_API, public_api, request)
@public_endpoints_bp.route('/getdata/<name>')
def get_block_data(name):
# Share data for a block if we have it
return getblocks.public_get_block_data(client_API, public_api, name)
@public_endpoints_bp.route('/www/<path:path>')
def www_public(path):
# A way to share files directly over your .onion
if not config.get("www.public.run", True):
abort(403)
return send_from_directory(config.get('www.public.path', 'static-data/www/public/'), path)
@public_endpoints_bp.route('/ping')
def ping():
# Endpoint to test if nodes are up
return Response("pong!")
@public_endpoints_bp.route('/pex')
def peer_exchange():
response = ','.join(client_API._core.listAdders(recent=3600))
if len(response) == 0:
response = ''
return Response(response)
@public_endpoints_bp.route('/announce', methods=['post'])
def accept_announce():
resp = httpapi.miscpublicapi.announce(client_API, request)
return resp
@public_endpoints_bp.route('/upload', methods=['post'])
def upload():
'''Accept file uploads. In the future this will be done more often than on creation
to speed up block sync
'''
return upload.accept_upload(client_API, request)

View File

@ -1,6 +1,25 @@
'''
Onionr - Private P2P Communication
Process incoming requests to the client api server to validate they are legitimate
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import hmac
from flask import Blueprint, request, abort
from onionrservices import httpheaders
# Be extremely mindful of this. These are endpoints available without a password
whitelist_endpoints = ('siteapi.site', 'www', 'staticfiles.onionrhome', 'staticfiles.homedata',
'staticfiles.board', 'staticfiles.profiles',
@ -9,28 +28,34 @@ whitelist_endpoints = ('siteapi.site', 'www', 'staticfiles.onionrhome', 'staticf
'staticfiles.mail', 'staticfiles.mailindex', 'staticfiles.friends', 'staticfiles.friendsindex',
'staticfiles.clandestine', 'staticfiles.clandestineIndex')
@app.before_request
def validateRequest():
'''Validate request has set password and is the correct hostname'''
# For the purpose of preventing DNS rebinding attacks
if request.host != '%s:%s' % (self.host, self.bindPort):
abort(403)
if request.endpoint in whitelist_endpoints:
return
try:
if not hmac.compare_digest(request.headers['token'], self.clientToken):
if not hmac.compare_digest(request.form['token'], self.clientToken):
abort(403)
except KeyError:
if not hmac.compare_digest(request.form['token'], self.clientToken):
abort(403)
class ClientAPISecurity:
def __init__(self, client_api):
client_api_security_bp = Blueprint('clientapisecurity', __name__)
self.client_api_security_bp = client_api_security_bp
self.client_api = client_api
@app.after_request
def afterReq(resp):
# Security headers
resp = httpheaders.set_default_onionr_http_headers(resp)
if request.endpoint == 'site':
resp.headers['Content-Security-Policy'] = "default-src 'none'; style-src data: 'unsafe-inline'; img-src data:"
else:
resp.headers['Content-Security-Policy'] = "default-src 'none'; script-src 'self'; object-src 'none'; style-src 'self'; img-src 'self'; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'self'"
return resp
@client_api_security_bp.before_app_request
def validate_request():
'''Validate request has set password and is the correct hostname'''
# For the purpose of preventing DNS rebinding attacks
if request.host != '%s:%s' % (client_api.host, client_api.bindPort):
abort(403)
if request.endpoint in whitelist_endpoints:
return
try:
if not hmac.compare_digest(request.headers['token'], client_api.clientToken):
if not hmac.compare_digest(request.form['token'], client_api.clientToken):
abort(403)
except KeyError:
if not hmac.compare_digest(request.form['token'], client_api.clientToken):
abort(403)
@client_api_security_bp.after_app_request
def after_req(resp):
# Security headers
resp = httpheaders.set_default_onionr_http_headers(resp)
if request.endpoint == 'site':
resp.headers['Content-Security-Policy'] = "default-src 'none'; style-src data: 'unsafe-inline'; img-src data:"
else:
resp.headers['Content-Security-Policy'] = "default-src 'none'; script-src 'self'; object-src 'none'; style-src 'self'; img-src 'self'; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'self'"
return resp

View File

@ -0,0 +1,50 @@
'''
Onionr - Private P2P Communication
Process incoming requests to the public api server for certain attacks
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
from flask import Blueprint, request, abort
from onionrservices import httpheaders
from onionrutils import epoch
class PublicAPISecurity:
def __init__(self, public_api):
public_api_security_bp = Blueprint('publicapisecurity', __name__)
self.public_api_security_bp = public_api_security_bp
@public_api_security_bp.before_app_request
def validate_request():
'''Validate request has the correct hostname'''
# If high security level, deny requests to public (HS should be disabled anyway for Tor, but might not be for I2P)
if public_api.config.get('general.security_level', default=1) > 0:
abort(403)
if type(public_api.torAdder) is None and type(public_api.i2pAdder) is None:
# abort if our hs addresses are not known
abort(403)
if request.host not in (public_api.i2pAdder, public_api.torAdder):
# Disallow connection if wrong HTTP hostname, in order to prevent DNS rebinding attacks
abort(403)
public_api.hitCount += 1 # raise hit count for valid requests
@public_api_security_bp.after_app_request
def send_headers(resp):
'''Send api, access control headers'''
resp = httpheaders.set_default_onionr_http_headers(resp)
# Network API version
resp.headers['X-API'] = public_api.API_VERSION
public_api.lastRequest = epoch.get_rounded_epoch(roundS=5)
return resp

View File

@ -23,7 +23,7 @@ from flask import Flask
import core, logger, httpapi
import onionrexceptions
from netcontroller import getOpenPort
import api
from httpapi import apiutils
from onionrutils import stringvalidators, basicrequests, bytesconverter
from . import httpheaders
@ -40,7 +40,7 @@ class ConnectionServer:
socks = core_inst.config.get('tor.socksport') # Load config for Tor socks port for proxy
service_app = Flask(__name__) # Setup Flask app for server.
service_port = getOpenPort()
service_ip = api.setBindIP()
service_ip = apiutils.setbindip.set_bind_IP(core_inst=self.core_inst)
http_server = WSGIServer(('127.0.0.1', service_port), service_app, log=None)
core_inst.onionrInst.communicatorInst.service_greenlets.append(http_server)