kev/Onionr
1
0
Fork 0
Code Issues 24 Pull Requests Projects Releases 2 Wiki Activity

Merge branch 'api2' into dbstorage

Browse Source
This commit is contained in:
Kevin Froman 2018-12-31 13:44:38 -06:00
commit fd4f1967e8
14 changed files with 541 additions and 619 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

766
onionr/api.py
View File

@ -19,55 +19,202 @@
'''
import flask
from flask import request, Response, abort, send_from_directory
from multiprocessing import Process
from gevent.pywsgi import WSGIServer
import sys, random, threading, hmac, hashlib, base64, time, math, os, json
import core
from onionrblockapi import Block
import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config, onionr
def guessMime(path):
'''
Guesses the mime type of a file from the input filename
'''
mimetypes = {
'html' : 'text/html',
'js' : 'application/javascript',
'css' : 'text/css',
'png' : 'image/png',
'jpg' : 'image/jpeg'
}
for mimetype in mimetypes:
if path.endswith('.%s' % mimetype):
return mimetypes[mimetype]
return 'text/plain'
def setBindIP(filePath):
'''Set a random localhost IP to a specified file (intended for private or public API localhost IPs)'''
hostOctets = [str(127), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF))]
data = '.'.join(hostOctets)
with open(filePath, 'w') as bindFile:
bindFile.write(data)
return data
class PublicAPI:
'''
The new client api server, isolated from the public api
'''
def __init__(self, clientAPI):
assert isinstance(clientAPI, API)
app = flask.Flask('PublicAPI')
self.i2pEnabled = config.get('i2p.host', False)
self.hideBlocks = [] # Blocks to be denied sharing
self.host = setBindIP(clientAPI._core.publicApiHostFile)
self.torAdder = clientAPI._core.hsAddress
self.i2pAdder = clientAPI._core.i2pAddress
self.bindPort = config.get('client.public.port')
logger.info('Running public api on %s:%s' % (self.host, self.bindPort))
@app.before_request
def validateRequest():
'''Validate request has the correct hostname'''
if type(self.torAdder) is None and type(self.i2pAdder) is None:
# abort if our hs addresses are not known
abort(403)
if request.host not in (self.i2pAdder, self.torAdder):
abort(403)
@app.after_request
def sendHeaders(resp):
'''Send api, access control headers'''
resp.headers['Date'] = 'Thu, 1 Jan 1970 00:00:00 GMT' # Clock info is probably useful to attackers. Set to unix epoch.
resp.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'"
resp.headers['X-Frame-Options'] = 'deny'
resp.headers['X-Content-Type-Options'] = "nosniff"
resp.headers['X-API'] = onionr.API_VERSION
return resp
@app.route('/')
def banner():
try:
with open('static-data/index.html', 'r') as html:
resp = Response(html.read(), mimetype='text/html')
except FileNotFoundError:
resp = Response("")
return resp
@app.route('/getblocklist')
def getBlockList():
bList = clientAPI._core.getBlockList()
for b in self.hideBlocks:
if b in bList:
bList.remove(b)
return Response('\n'.join(bList))
@app.route('/getdata/<name>')
def getBlockData(name):
resp = ''
data = name
if clientAPI._utils.validateHash(data):
if data not in self.hideBlocks:
if os.path.exists(clientAPI._core.dataDir + 'blocks/' + data + '.dat'):
block = Block(hash=data.encode(), core=clientAPI._core)
resp = base64.b64encode(block.getRaw().encode()).decode()
if len(resp) == 0:
abort(404)
resp = ""
return Response(resp)
@app.route('/www/<path:path>')
def wwwPublic(path):
if not config.get("www.public.run", True):
abort(403)
return send_from_directory(config.get('www.public.path', 'static-data/www/public/'), path)
@app.route('/ping')
def ping():
return Response("pong!")
@app.route('/getdbhash')
def getDBHash():
return Response(clientAPI._utils.getBlockDBHash())
@app.route('/pex')
def peerExchange():
response = ','.join(clientAPI._core.listAdders())
if len(response) == 0:
response = 'none'
return Response(response)
@app.route('/announce', methods=['post'])
def acceptAnnounce():
resp = 'failure'
powHash = ''
randomData = ''
newNode = ''
ourAdder = clientAPI._core.hsAddress.encode()
try:
newNode = request.form['node'].encode()
except KeyError:
logger.warn('No block specified for upload')
pass
else:
try:
randomData = request.form['random']
randomData = base64.b64decode(randomData)
except KeyError:
logger.warn('No random data specified for upload')
else:
nodes = newNode + clientAPI._core.hsAddress.encode()
nodes = clientAPI._core._crypto.blake2bHash(nodes)
powHash = clientAPI._core._crypto.blake2bHash(randomData + nodes)
try:
powHash = powHash.decode()
except AttributeError:
pass
if powHash.startswith('0000'):
try:
newNode = newNode.decode()
except AttributeError:
pass
if clientAPI._core.addAddress(newNode):
resp = 'Success'
else:
logger.warn(newNode.decode() + ' failed to meet POW: ' + powHash)
resp = Response(resp)
return resp
@app.route('/upload', methods=['post'])
def upload():
resp = 'failure'
try:
data = request.form['block']
except KeyError:
logger.warn('No block specified for upload')
pass
else:
if sys.getsizeof(data) < 100000000:
try:
if blockimporter.importBlockFromData(data, clientAPI._core):
resp = 'success'
else:
logger.warn('Error encountered importing uploaded block')
except onionrexceptions.BlacklistedBlock:
logger.debug('uploaded block is blacklisted')
pass
if resp == 'failure':
abort(400)
resp = Response(resp)
return resp
clientAPI.setPublicAPIInstance(self)
while self.torAdder == '':
clientAPI._core.refreshFirstStartVars()
self.torAdder = clientAPI._core.hsAddress
time.sleep(1)
self.httpServer = WSGIServer((self.host, self.bindPort), app, log=None)
self.httpServer.serve_forever()
class API:
'''
Main HTTP API (Flask)
Client HTTP api
'''
callbacks = {'public' : {}, 'private' : {}}
def validateToken(self, token):
'''
Validate that the client token matches the given token
'''
if len(self.clientToken) == 0:
logger.error("client password needs to be set")
return False
try:
if not hmac.compare_digest(self.clientToken, token):
return False
else:
return True
except TypeError:
return False
def guessMime(path):
'''
Guesses the mime type from the input filename
'''
mimetypes = {
'html' : 'text/html',
'js' : 'application/javascript',
'css' : 'text/css',
'png' : 'image/png',
'jpg' : 'image/jpeg'
}
for mimetype in mimetypes:
if path.endswith('.%s' % mimetype):
return mimetypes[mimetype]
return 'text/plain'
def __init__(self, debug, API_VERSION):
def __init__(self, onionrInst, debug, API_VERSION):
'''
Initialize the api server, preping variables for later use
@ -84,51 +231,50 @@ class API:
self._crypto = onionrcrypto.OnionrCrypto(self._core)
self._utils = onionrutils.OnionrUtils(self._core)
app = flask.Flask(__name__)
bindPort = int(config.get('client.port', 59496))
bindPort = int(config.get('client.client.port', 59496))
self.bindPort = bindPort
self.clientToken = config.get('client.webpassword')
self.timeBypassToken = base64.b16encode(os.urandom(32)).decode()
self.i2pEnabled = config.get('i2p.host', False)
self.hideBlocks = [] # Blocks to be denied sharing
with open(self._core.dataDir + 'time-bypass.txt', 'w') as bypass:
bypass.write(self.timeBypassToken)
if not debug and not self._developmentMode:
hostOctets = [str(127), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF))]
self.host = '.'.join(hostOctets)
else:
self.host = '127.0.0.1'
with open(self._core.dataDir + 'host.txt', 'w') as file:
file.write(self.host)
self.publicAPI = None # gets set when the thread calls our setter... bad hack but kinda necessary with flask
#threading.Thread(target=PublicAPI, args=(self,)).start()
self.host = setBindIP(self._core.privateApiHostFile)
logger.info('Running api on %s:%s' % (self.host, self.bindPort))
self.httpServer = ''
onionrInst.setClientAPIInst(self)
@app.before_request
def beforeReq():
'''
Simply define the request as not having yet failed, before every request.
'''
self.requestFailed = False
return
def validateRequest():
'''Validate request has set password and is the correct hostname'''
if request.host != '%s:%s' % (self.host, self.bindPort):
abort(403)
try:
if not hmac.compare_digest(request.headers['token'], self.clientToken):
abort(403)
except KeyError:
abort(403)
@app.after_request
def afterReq(resp):
if not self.requestFailed:
resp.headers['Access-Control-Allow-Origin'] = '*'
#else:
# resp.headers['server'] = 'Onionr'
resp.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'"
resp.headers['X-Frame-Options'] = 'deny'
resp.headers['X-Content-Type-Options'] = "nosniff"
resp.headers['X-API'] = API_VERSION
resp.headers['X-API'] = onionr.API_VERSION
resp.headers['Server'] = ''
resp.headers['Date'] = 'Thu, 1 Jan 1970 00:00:00 GMT' # Clock info is probably useful to attackers. Set to unix epoch.
return resp
@app.route('/site/<path:block>')
def site(block):
self.validateHost('private')
@app.route('/ping')
def ping():
return Response("pong!")
@app.route('/')
def hello():
return Response("hello client")
@app.route('/site/<name>')
def site():
bHash = block
resp = 'Not Found'
if self._core._utils.validateHash(bHash):
@ -137,467 +283,47 @@ class API:
resp = base64.b64decode(resp)
except:
pass
if resp == 'Not Found':
abourt(404)
return Response(resp)
@app.route('/www/private/<path:path>')
def www_private(path):
startTime = math.floor(time.time())
if request.args.get('timingToken') is None:
timingToken = ''
@app.route('/waitforshare/<name>', methods=['post'])
def waitforshare():
assert name.isalnum()
if name in self.publicAPI.hideBlocks:
self.publicAPI.hideBlocks.remove(name)
return Response("removed")
else:
timingToken = request.args.get('timingToken')
self.publicAPI.hideBlocks.append(name)
return Response("added")
if not config.get("www.private.run", True):
abort(403)
self.validateHost('private')
endTime = math.floor(time.time())
elapsed = endTime - startTime
if not hmac.compare_digest(timingToken, self.timeBypassToken):
if (elapsed < self._privateDelayTime) and config.get('www.private.timing_protection', True):
time.sleep(self._privateDelayTime - elapsed)
return send_from_directory(config.get('www.private.path', 'static-data/www/private/'), path)
@app.route('/www/public/<path:path>')
def www_public(path):
if not config.get("www.public.run", True):
abort(403)
self.validateHost('public')
return send_from_directory(config.get('www.public.path', 'static-data/www/public/'), path)
@app.route('/ui/<path:path>')
def ui_private(path):
startTime = math.floor(time.time())
'''
if request.args.get('timingToken') is None:
timingToken = ''
else:
timingToken = request.args.get('timingToken')
'''
if not config.get("www.ui.run", True):
abort(403)
if config.get("www.ui.private", True):
self.validateHost('private')
else:
self.validateHost('public')
'''
endTime = math.floor(time.time())
elapsed = endTime - startTime
if not hmac.compare_digest(timingToken, self.timeBypassToken):
if elapsed < self._privateDelayTime:
time.sleep(self._privateDelayTime - elapsed)
'''
mime = API.guessMime(path)
logger.debug('Serving %s (mime: %s)' % (path, mime))
return send_from_directory('static-data/www/ui/dist/', path)
@app.route('/client/')
def private_handler():
if request.args.get('timingToken') is None:
timingToken = ''
else:
timingToken = request.args.get('timingToken')
data = request.args.get('data')
@app.route('/shutdown')
def shutdown():
try:
data = data
except:
data = ''
startTime = math.floor(time.time())
action = request.args.get('action')
#if not self.debug:
token = request.args.get('token')
if not self.validateToken(token):
abort(403)
events.event('webapi_private', onionr = None, data = {'action' : action, 'data' : data, 'timingToken' : timingToken, 'token' : token})
self.validateHost('private')
if action == 'hello':
resp = Response('Hello, World! ' + request.host)
elif action == 'getIP':
resp = Response(self.host)
elif action == 'waitForShare':
if self._core._utils.validateHash(data):
if data not in self.hideBlocks:
self.hideBlocks.append(data)
else:
self.hideBlocks.remove(data)
resp = "success"
else:
resp = "failed to validate hash"
elif action == 'shutdown':
# request.environ.get('werkzeug.server.shutdown')()
self.http_server.stop()
resp = Response('Goodbye')
elif action == 'ping':
resp = Response('pong')
elif action == 'info':
resp = Response(json.dumps({'pubkey' : self._core._crypto.pubKey, 'host' : self._core.hsAddress}), mimetype='text/plain')
elif action == "insertBlock":
response = {'success' : False, 'reason' : 'An unknown error occurred'}
if not ((data is None) or (len(str(data).strip()) == 0)):
try:
decoded = json.loads(data)
block = Block()
sign = False
for key in decoded:
val = decoded[key]
key = key.lower()
if key == 'type':
block.setType(val)
elif key in ['body', 'content']:
block.setContent(val)
elif key == 'parent':
block.setParent(val)
elif key == 'sign':
sign = (str(val).lower() == 'true')
hash = block.save(sign = sign)
if not hash is False:
response['success'] = True
response['hash'] = hash
response['reason'] = 'Successfully wrote block to file'
else:
response['reason'] = 'Failed to save the block'
except Exception as e:
logger.warn('insertBlock api request failed', error = e)
logger.debug('Here\'s the request: %s' % data)
else:
response = {'success' : False, 'reason' : 'Missing `data` parameter.', 'blocks' : {}}
resp = Response(json.dumps(response))
elif action == 'searchBlocks':
response = {'success' : False, 'reason' : 'An unknown error occurred', 'blocks' : {}}
if not ((data is None) or (len(str(data).strip()) == 0)):
try:
decoded = json.loads(data)
type = None
signer = None
signed = None
parent = None
reverse = False
limit = None
for key in decoded:
val = decoded[key]
key = key.lower()
if key == 'type':
type = str(val)
elif key == 'signer':
if isinstance(val, list):
signer = val
else:
signer = str(val)
elif key == 'signed':
signed = (str(val).lower() == 'true')
elif key == 'parent':
parent = str(val)
elif key == 'reverse':
reverse = (str(val).lower() == 'true')
elif key == 'limit':
limit = 10000
if val is None:
val = limit
limit = min(limit, int(val))
blockObjects = Block.getBlocks(type = type, signer = signer, signed = signed, parent = parent, reverse = reverse, limit = limit)
logger.debug('%s results for query %s' % (len(blockObjects), decoded))
blocks = list()
for block in blockObjects:
blocks.append({
'hash' : block.getHash(),
'type' : block.getType(),
'content' : block.getContent(),
'signature' : block.getSignature(),
'signedData' : block.getSignedData(),
'signed' : block.isSigned(),
'valid' : block.isValid(),
'date' : (int(block.getDate().strftime("%s")) if not block.getDate() is None else None),
'parent' : (block.getParent().getHash() if not block.getParent() is None else None),
'metadata' : block.getMetadata(),
'header' : block.getHeader()
})
response['success'] = True
response['blocks'] = blocks
response['reason'] = 'Success'
except Exception as e:
logger.warn('searchBlock api request failed', error = e)
logger.debug('Here\'s the request: %s' % data)
else:
response = {'success' : False, 'reason' : 'Missing `data` parameter.', 'blocks' : {}}
resp = Response(json.dumps(response), mimetype='text/plain')
elif action in API.callbacks['private']:
resp = Response(str(getCallback(action, scope = 'private')(request)), mimetype='text/plain')
else:
resp = Response('invalid command')
endTime = math.floor(time.time())
elapsed = endTime - startTime
# if bypass token not used, delay response to prevent timing attacks
if not hmac.compare_digest(timingToken, self.timeBypassToken):
if elapsed < self._privateDelayTime:
time.sleep(self._privateDelayTime - elapsed)
return resp
@app.route('/')
def banner():
self.validateHost('public')
try:
with open('static-data/index.html', 'r') as html:
resp = Response(html.read(), mimetype='text/html')
except FileNotFoundError:
resp = Response("")
return resp
@app.route('/public/upload/', methods=['POST'])
def blockUpload():
self.validateHost('public')
resp = 'failure'
try:
data = request.form['block']
except KeyError:
logger.warn('No block specified for upload')
self.publicAPI.httpServer.stop()
self.httpServer.stop()
except AttributeError:
pass
else:
if sys.getsizeof(data) < 100000000:
try:
if blockimporter.importBlockFromData(data, self._core):
resp = 'success'
else:
logger.warn('Error encountered importing uploaded block')
except onionrexceptions.BlacklistedBlock:
logger.debug('uploaded block is blacklisted')
pass
return Response("bye")
resp = Response(resp)
return resp
self.httpServer = WSGIServer((self.host, bindPort), app, log=None)
self.httpServer.serve_forever()
@app.route('/public/announce/', methods=['POST'])
def acceptAnnounce():
self.validateHost('public')
resp = 'failure'
powHash = ''
randomData = ''
newNode = ''
ourAdder = self._core.hsAddress.encode()
try:
newNode = request.form['node'].encode()
except KeyError:
logger.warn('No block specified for upload')
pass
else:
try:
randomData = request.form['random']
randomData = base64.b64decode(randomData)
except KeyError:
logger.warn('No random data specified for upload')
else:
nodes = newNode + self._core.hsAddress.encode()
nodes = self._core._crypto.blake2bHash(nodes)
powHash = self._core._crypto.blake2bHash(randomData + nodes)
try:
powHash = powHash.decode()
except AttributeError:
pass
if powHash.startswith('0000'):
try:
newNode = newNode.decode()
except AttributeError:
pass
if self._core.addAddress(newNode):
resp = 'Success'
else:
logger.warn(newNode.decode() + ' failed to meet POW: ' + powHash)
resp = Response(resp)
return resp
@app.route('/public/')
def public_handler():
# Public means it is publicly network accessible
self.validateHost('public')
if config.get('general.security_level') != 0:
abort(403)
action = request.args.get('action')
requestingPeer = request.args.get('myID')
data = request.args.get('data')
try:
data = data
except:
data = ''
events.event('webapi_public', onionr = None, data = {'action' : action, 'data' : data, 'requestingPeer' : requestingPeer, 'request' : request})
if action == 'firstConnect':
pass
elif action == 'ping':
resp = Response("pong!")
elif action == 'getSymmetric':
resp = Response(self._crypto.generateSymmetric())
elif action == 'getDBHash':
resp = Response(self._utils.getBlockDBHash())
elif action == 'getBlockHashes':
bList = self._core.getBlockList()
for b in self.hideBlocks:
if b in bList:
bList.remove(b)
resp = Response('\n'.join(bList))
# setData should be something the communicator initiates, not this api
elif action == 'getData':
resp = ''
if self._utils.validateHash(data):
if data not in self.hideBlocks:
if os.path.exists(self._core.dataDir + 'blocks/' + data + '.dat'):
block = Block(hash=data.encode(), core=self._core)
resp = base64.b64encode(block.getRaw().encode()).decode()
if len(resp) == 0:
abort(404)
resp = ""
resp = Response(resp)
elif action == 'pex':
response = ','.join(self._core.listAdders())
if len(response) == 0:
response = 'none'
resp = Response(response)
elif action == 'kex':
peers = self._core.listPeers(getPow=True)
response = ','.join(peers)
resp = Response(response)
elif action in API.callbacks['public']:
resp = Response(str(getCallback(action, scope = 'public')(request)))
else:
resp = Response("")
return resp
@app.errorhandler(404)
def notfound(err):
self.requestFailed = True
resp = Response("")
return resp
@app.errorhandler(403)
def authFail(err):
self.requestFailed = True
resp = Response("403")
return resp
@app.errorhandler(401)
def clientError(err):
self.requestFailed = True
resp = Response("Invalid request")
return resp
logger.info('Starting client on ' + self.host + ':' + str(bindPort), timestamp=False)
def setPublicAPIInstance(self, inst):
assert isinstance(inst, PublicAPI)
self.publicAPI = inst
def validateToken(self, token):
'''
Validate that the client token matches the given token
'''
if len(self.clientToken) == 0:
logger.error("client password needs to be set")
return False
try:
while len(self._core.hsAddress) == 0:
self._core.refreshFirstStartVars()
time.sleep(0.5)
self.http_server = WSGIServer((self.host, bindPort), app, log=None)
self.http_server.serve_forever()
except KeyboardInterrupt:
pass
except Exception as e:
logger.error(str(e))
logger.fatal('Failed to start client on ' + self.host + ':' + str(bindPort) + ', exiting...')
def validateHost(self, hostType):
'''
Validate various features of the request including:
If private (/client/), is the host header local?
If public (/public/), is the host header onion or i2p?
Was X-Request-With used?
'''
if self.debug:
return
# Validate host header, to protect against DNS rebinding attacks
host = self.host
if hostType == 'private':
if not request.host.startswith('127') and not self._utils.checkIsIP(request.host):
abort(403)
elif hostType == 'public':
if not request.host.endswith('onion') and not request.host.endswith('i2p'):
abort(403)
# Validate x-requested-with, to protect against CSRF/metadata leaks
if not self.i2pEnabled and request.host.endswith('i2p'):
abort(403)
'''
if not self._developmentMode:
try:
request.headers['X-Requested-With']
except:
pass
# we exit rather than abort to avoid fingerprinting
logger.debug('Avoiding fingerprinting, exiting...')
#sys.exit(1)
'''
def setCallback(action, callback, scope = 'public'):
if not scope in API.callbacks:
if not hmac.compare_digest(self.clientToken, token):
return False
else:
return True
except TypeError:
return False
API.callbacks[scope][action] = callback
return True
def removeCallback(action, scope = 'public'):
if (not scope in API.callbacks) or (not action in API.callbacks[scope]):
return False
del API.callbacks[scope][action]
return True
def getCallback(action, scope = 'public'):
if (not scope in API.callbacks) or (not action in API.callbacks[scope]):
return None
return API.callbacks[scope][action]
def getCallbacks(scope = None):
if (not scope is None) and (scope in API.callbacks):
return API.callbacks[scope]
return API.callbacks

75
onionr/apimanager.py Normal file
View File

@ -0,0 +1,75 @@
'''
Onionr - P2P Anonymous Storage Network
Handles api data exchange, interfaced by both public and client http api
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import config, apipublic, apiprivate, core, socket, random, threading, time
config.reload()
PRIVATE_API_VERSION = 0
PUBLIC_API_VERSION = 1
DEV_MODE = config.get('general.dev_mode')
def getOpenPort():
'''Get a random open port'''
p = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
p.bind(("127.0.0.1",0))
p.listen(1)
port = p.getsockname()[1]
p.close()
return port
def getRandomLocalIP():
'''Get a random local ip address'''
hostOctets = [str(127), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF))]
host = '.'.join(hostOctets)
return host
class APIManager:
def __init__(self, coreInst):
assert isinstance(coreInst, core.Core)
self.core = coreInst
self.utils = coreInst._utils
self.crypto = coreInst._crypto
# if this gets set to true, both the public and private apis will shutdown
self.shutdown = False
publicIP = '127.0.0.1'
privateIP = '127.0.0.1'
if DEV_MODE:
# set private and local api servers bind IPs to random localhost (127.x.x.x), make sure not the same
privateIP = getRandomLocalIP()
while True:
publicIP = getRandomLocalIP()
if publicIP != privateIP:
break
# Make official the IPs and Ports
self.publicIP = publicIP
self.privateIP = privateIP
self.publicPort = config.get('client.port', 59496)
self.privatePort = config.get('client.port', 59496)
# Run the API servers in new threads
self.publicAPI = apipublic.APIPublic(self)
self.privateAPI = apiprivate.APIPrivate(self)
threading.Thread(target=self.publicAPI.run).start()
threading.Thread(target=self.privateAPI.run).start()
while not self.shutdown:
time.sleep(1)

32
onionr/apiprivate.py Normal file
View File

@ -0,0 +1,32 @@
'''
Onionr - P2P Anonymous Storage Network
Handle incoming commands from the client. Intended for localhost use
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import flask, apimanager
from flask import request, Response, abort, send_from_directory
from gevent.pywsgi import WSGIServer
class APIPrivate:
def __init__(self, managerInst):
assert isinstance(managerInst, apimanager.APIManager)
self.app = flask.Flask(__name__) # The flask application, which recieves data from the greenlet wsgiserver
self.httpServer = WSGIServer((managerInst.privateIP, managerInst.privatePort), self.app, log=None)
def run(self):
self.httpServer.serve_forever()
return

41
onionr/apipublic.py Normal file
View File

@ -0,0 +1,41 @@
'''
Onionr - P2P Anonymous Storage Network
Handle incoming commands from other Onionr nodes, over HTTP
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import flask, apimanager
from flask import request, Response, abort, send_from_directory
from gevent.pywsgi import WSGIServer
class APIPublic:
def __init__(self, managerInst):
assert isinstance(managerInst, apimanager.APIManager)
app = flask.Flask(__name__)
@app.route('/')
def banner():
try:
with open('static-data/index.html', 'r') as html:
resp = Response(html.read(), mimetype='text/html')
except FileNotFoundError:
resp = Response("")
return resp
self.httpServer = WSGIServer((managerInst.publicIP, managerInst.publicPort), app)
def run(self):
self.httpServer.serve_forever()
return

23
onionr/communicator2.py
View File

@ -22,6 +22,7 @@
import sys, os, core, config, json, requests, time, logger, threading, base64, onionr, uuid
import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block
import onionrdaemontools, onionrsockets, onionrchat, onionr, onionrproofs
import binascii
from dependencies import secrets
from defusedxml import minidom
@ -101,6 +102,7 @@ class OnionrCommunicatorDaemon:
OnionrCommunicatorTimers(self, self.daemonTools.cooldownPeer, 30, requiresPeer=True)
OnionrCommunicatorTimers(self, self.uploadBlock, 10, requiresPeer=True, maxThreads=1)
OnionrCommunicatorTimers(self, self.daemonCommands, 6, maxThreads=1)
OnionrCommunicatorTimers(self, self.detectAPICrash, 5, maxThreads=1)
deniableBlockTimer = OnionrCommunicatorTimers(self, self.daemonTools.insertDeniableBlock, 180, requiresPeer=True, maxThreads=1)
netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600)
@ -178,14 +180,14 @@ class OnionrCommunicatorDaemon:
break
else:
continue
newDBHash = self.peerAction(peer, 'getDBHash') # get their db hash
newDBHash = self.peerAction(peer, 'getdbhash') # get their db hash
if newDBHash == False or not self._core._utils.validateHash(newDBHash):
continue # if request failed, restart loop (peer is added to offline peers automatically)
triedPeers.append(peer)
if newDBHash != self._core.getAddressInfo(peer, 'DBHash'):
self._core.setAddressInfo(peer, 'DBHash', newDBHash)
try:
newBlocks = self.peerAction(peer, 'getBlockHashes') # get list of new block hashes
newBlocks = self.peerAction(peer, 'getblocklist') # get list of new block hashes
except Exception as error:
logger.warn('Could not get new blocks from %s.' % peer, error = error)
newBlocks = False
@ -224,13 +226,16 @@ class OnionrCommunicatorDaemon:
self.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block
logger.info("Attempting to download %s..." % blockHash)
peerUsed = self.pickOnlinePeer()
content = self.peerAction(peerUsed, 'getData', data=blockHash) # block content from random peer (includes metadata)
content = self.peerAction(peerUsed, 'getdata/' + blockHash) # block content from random peer (includes metadata)
if content != False and len(content) > 0:
try:
content = content.encode()
except AttributeError:
pass
content = base64.b64decode(content) # content is base64 encoded in transport
try:
content = base64.b64decode(content) # content is base64 encoded in transport
except binascii.Error:
pass
realHash = self._core._crypto.sha3Hash(content)
try:
realHash = realHash.decode() # bytes on some versions for some reason
@ -421,7 +426,7 @@ class OnionrCommunicatorDaemon:
if len(peer) == 0:
return False
#logger.debug('Performing ' + action + ' with ' + peer + ' on port ' + str(self.proxyPort))
url = 'http://' + peer + '/public/?action=' + action
url = 'http://%s/%s' % (peer, action)
if len(data) > 0:
url += '&data=' + data
@ -518,7 +523,7 @@ class OnionrCommunicatorDaemon:
if peer in triedPeers:
continue
triedPeers.append(peer)
url = 'http://' + peer + '/public/upload/'
url = 'http://' + peer + '/upload'
data = {'block': block.Block(bl).getRaw()}
proxyType = ''
if peer.endswith('.onion'):
@ -527,7 +532,7 @@ class OnionrCommunicatorDaemon:
proxyType = 'i2p'
logger.info("Uploading block to " + peer)
if not self._core._utils.doPostRequest(url, data=data, proxyType=proxyType) == False:
self._core._utils.localCommand('waitForShare', data=bl)
self._core._utils.localCommand('waitforshare/' + bl)
finishedUploads.append(bl)
break
for x in finishedUploads:
@ -544,9 +549,9 @@ class OnionrCommunicatorDaemon:
def detectAPICrash(self):
'''exit if the api server crashes/stops'''
if self._core._utils.localCommand('ping', silent=False) != 'pong':
if self._core._utils.localCommand('ping', silent=False) not in ('pong', 'pong!'):
for i in range(5):
if self._core._utils.localCommand('ping') == 'pong':
if self._core._utils.localCommand('ping') in ('pong', 'pong!'):
break # break for loop
time.sleep(1)
else:

11
onionr/core.py
View File

@ -50,8 +50,11 @@ class Core:
self.blockDB = self.dataDir + 'blocks.db'
self.blockDataDB = self.dataDir + 'block-data.db'
self.blockDataLocation = self.dataDir + 'blocks/'
self.publicApiHostFile = self.dataDir + 'public-host.txt'
self.privateApiHostFile = self.dataDir + 'private-host.txt'
self.addressDB = self.dataDir + 'address.db'
self.hsAddress = ''
self.i2pAddress = config.get('i2p.ownAddr', None)
self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt'
self.bootstrapList = []
self.requirements = onionrvalues.OnionrValues()
@ -152,7 +155,7 @@ class Core:
if address == config.get('i2p.ownAddr', None) or address == self.hsAddress:
return False
if type(address) is type(None) or len(address) == 0:
if type(address) is None or len(address) == 0:
return False
if self._utils.validateID(address):
conn = sqlite3.connect(self.addressDB, timeout=10)
@ -258,7 +261,7 @@ class Core:
return
conn = sqlite3.connect(self.blockDB, timeout=10)
c = conn.cursor()
currentTime = self._utils.getEpoch()
currentTime = self._utils.getEpoch() + self._crypto.secrets.randbelow(301)
if selfInsert or dataSaved:
selfInsert = 1
else:
@ -761,7 +764,7 @@ class Core:
metadata['meta'] = jsonMeta
metadata['sig'] = signature
metadata['signer'] = signer
metadata['time'] = self._utils.getRoundedEpoch() + self._crypto.secrets.randbelow(301)
metadata['time'] = self._utils.getRoundedEpoch()
# ensure expire is integer and of sane length
if type(expire) is not type(None):
@ -774,7 +777,7 @@ class Core:
if payload != False:
retData = self.setData(payload)
# Tell the api server through localCommand to wait for the daemon to upload this block to make stastical analysis more difficult
self._utils.localCommand('waitForShare', data=retData)
self._utils.localCommand('waitforshare/' + retData)
self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
#self.setBlockType(retData, meta['type'])
self._utils.processBlockMetadata(retData)

17
onionr/netcontroller.py
View File

@ -18,10 +18,19 @@
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import subprocess, os, random, sys, logger, time, signal, config, base64
import subprocess, os, random, sys, logger, time, signal, config, base64, socket
from stem.control import Controller
from onionrblockapi import Block
from dependencies import secrets
def getOpenPort():
# taken from (but modified) https://stackoverflow.com/a/2838309
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("127.0.0.1",0))
s.listen(1)
port = s.getsockname()[1]
s.close()
return port
class NetController:
'''
This class handles hidden service setup on Tor and I2P
@ -37,7 +46,7 @@ class NetController:
self.torConfigLocation = self.dataDir + 'torrc'
self.readyState = False
self.socksPort = random.randint(1024, 65535)
self.socksPort = getOpenPort()
self.hsPort = hsPort
self._torInstnace = ''
self.myID = ''
@ -68,7 +77,7 @@ class NetController:
hsVer = '# v2 onions'
if config.get('tor.v3onions'):
hsVer = 'HiddenServiceVersion 3'
logger.debug('Using v3 onions :)')
logger.debug('Using v3 onions')
if os.path.exists(self.torConfigLocation):
os.remove(self.torConfigLocation)
@ -78,7 +87,7 @@ class NetController:
config.set('tor.controlpassword', plaintext, savefile=True)
config.set('tor.socksport', self.socksPort, savefile=True)
controlPort = random.randint(1025, 65535)
controlPort = getOpenPort()
config.set('tor.controlPort', controlPort, savefile=True)

128
onionr/onionr.py
View File

@ -20,7 +20,6 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import sys
if sys.version_info[0] == 2 or sys.version_info[1] < 5:
print('Error, Onionr requires Python 3.5+')
@ -30,6 +29,7 @@ import webbrowser
from threading import Thread
import api, core, config, logger, onionrplugins as plugins, onionrevents as events
import onionrutils
import netcontroller
from netcontroller import NetController
from onionrblockapi import Block
import onionrproofs, onionrexceptions, onionrusers
@ -67,8 +67,12 @@ class Onionr:
data_exists = Onionr.setupConfig(self.dataDir, self = self)
self.onionrCore = core.Core()
#self.deleteRunFiles()
self.onionrUtils = onionrutils.OnionrUtils(self.onionrCore)
self.clientAPIInst = '' # Client http api instance
self.publicAPIInst = '' # Public http api instance
# Handle commands
self.debug = False # Whole application debugging
@ -105,11 +109,12 @@ class Onionr:
# Get configuration
if type(config.get('client.webpassword')) is type(None):
config.set('client.webpassword', base64.b16encode(os.urandom(32)).decode('utf-8'), savefile=True)
if type(config.get('client.port')) is type(None):
randomPort = 0
while randomPort < 1024:
randomPort = self.onionrCore._crypto.secrets.randbelow(65535)
config.set('client.port', randomPort, savefile=True)
if type(config.get('client.client.port')) is type(None):
randomPort = netcontroller.getOpenPort()
config.set('client.client.port', randomPort, savefile=True)
if type(config.get('client.public.port')) is type(None):
randomPort = netcontroller.getOpenPort()
config.set('client.public.port', randomPort, savefile=True)
if type(config.get('client.participate')) is type(None):
config.set('client.participate', True, savefile=True)
if type(config.get('client.api_version')) is type(None):
@ -176,6 +181,7 @@ class Onionr:
'getfile': self.getFile,
'listconn': self.listConn,
'list-conn': self.listConn,
'import-blocks': self.onionrUtils.importNewBlocks,
'importblocks': self.onionrUtils.importNewBlocks,
@ -395,6 +401,16 @@ class Onionr:
logger.info('Syntax: friend add/remove/list [address]')
def deleteRunFiles(self):
try:
os.remove(self.onionrCore.publicApiHostFile)
except FileNotFoundError:
pass
try:
os.remove(self.onionrCore.privateApiHostFile)
except FileNotFoundError:
pass
def banBlock(self):
try:
ban = sys.argv[2]
@ -691,6 +707,13 @@ class Onionr:
os.remove('.onionr-lock')
except FileNotFoundError:
pass
def setClientAPIInst(self, inst):
self.clientAPIInst = inst
def getClientApi(self):
while self.clientAPIInst == '':
time.sleep(0.5)
return self.clientAPIInst
def daemon(self):
'''
@ -704,63 +727,66 @@ class Onionr:
logger.debug('Runcheck file found on daemon start, deleting in advance.')
os.remove('data/.runcheck')
apiThread = Thread(target = api.API, args = (self.debug, API_VERSION))
apiThread.start()
Thread(target=api.API, args=(self, self.debug, API_VERSION)).start()
Thread(target=api.PublicAPI, args=[self.getClientApi()]).start()
try:
time.sleep(3)
time.sleep(0)
except KeyboardInterrupt:
logger.debug('Got keyboard interrupt, shutting down...')
time.sleep(1)
self.onionrUtils.localCommand('shutdown')
apiHost = ''
while apiHost == '':
try:
with open(self.onionrCore.publicApiHostFile, 'r') as hostFile:
apiHost = hostFile.read()
except FileNotFoundError:
pass
time.sleep(0.5)
Onionr.setupConfig('data/', self = self)
if self._developmentMode:
logger.warn('DEVELOPMENT MODE ENABLED (LESS SECURE)', timestamp = False)
net = NetController(config.get('client.public.port', 59497), apiServerIP=apiHost)
logger.debug('Tor is starting...')
if not net.startTor():
self.onionrUtils.localCommand('shutdown')
sys.exit(1)
if len(net.myID) > 0 and config.get('general.security_level') == 0:
logger.debug('Started .onion service: %s' % (logger.colors.underline + net.myID))
else:
apiHost = '127.0.0.1'
if apiThread.isAlive():
try:
with open(self.onionrCore.dataDir + 'host.txt', 'r') as hostFile:
apiHost = hostFile.read()
except FileNotFoundError:
pass
Onionr.setupConfig('data/', self = self)
logger.debug('.onion service disabled')
logger.debug('Using public key: %s' % (logger.colors.underline + self.onionrCore._crypto.pubKey))
time.sleep(1)
if self._developmentMode:
logger.warn('DEVELOPMENT MODE ENABLED (LESS SECURE)', timestamp = False)
net = NetController(config.get('client.port', 59496), apiServerIP=apiHost)
logger.debug('Tor is starting...')
if not net.startTor():
self.onionrUtils.localCommand('shutdown')
sys.exit(1)
if len(net.myID) > 0 and config.get('general.security_level') == 0:
logger.debug('Started .onion service: %s' % (logger.colors.underline + net.myID))
else:
logger.debug('.onion service disabled')
logger.debug('Using public key: %s' % (logger.colors.underline + self.onionrCore._crypto.pubKey))
time.sleep(1)
# TODO: make runable on windows
communicatorProc = subprocess.Popen([communicatorDaemon, 'run', str(net.socksPort)])
# TODO: make runable on windows
communicatorProc = subprocess.Popen([communicatorDaemon, 'run', str(net.socksPort)])
# print nice header thing :)
if config.get('general.display_header', True):
self.header()
# print nice header thing :)
if config.get('general.display_header', True):
self.header()
# print out debug info
self.version(verbosity = 5, function = logger.debug)
logger.debug('Python version %s' % platform.python_version())
# print out debug info
self.version(verbosity = 5, function = logger.debug)
logger.debug('Python version %s' % platform.python_version())
logger.debug('Started communicator.')
logger.debug('Started communicator.')
events.event('daemon_start', onionr = self)
try:
while True:
time.sleep(5)
events.event('daemon_start', onionr = self)
try:
while True:
time.sleep(5)
# Break if communicator process ends, so we don't have left over processes
if communicatorProc.poll() is not None:
break
except KeyboardInterrupt:
self.onionrCore.daemonQueueAdd('shutdown')
self.onionrUtils.localCommand('shutdown')
# Break if communicator process ends, so we don't have left over processes
if communicatorProc.poll() is not None:
break
except KeyboardInterrupt:
self.onionrCore.daemonQueueAdd('shutdown')
self.onionrUtils.localCommand('shutdown')
time.sleep(3)
self.deleteRunFiles()
net.killTor()
return
def killDaemon(self):

2
onionr/onionrdaemontools.py
View File

@ -45,7 +45,7 @@ class DaemonTools:
ourID = self.daemon._core.hsAddress.strip()
url = 'http://' + peer + '/public/announce/'
url = 'http://' + peer + '/announce'
data = {'node': ourID}
combinedNodes = ourID + peer

38
onionr/onionrpeers.py
View File

@ -79,27 +79,29 @@ def peerCleanup(coreInst):
logger.info('Cleaning peers...')
config.reload()
minScore = int(config.get('peers.minimum_score', -100))
maxPeers = int(config.get('peers.max_stored', 5000))
adders = getScoreSortedPeerList(coreInst)
adders.reverse()
if len(adders) > 1:
for address in adders:
# Remove peers that go below the negative score
if PeerProfiles(address, coreInst).score < minScore:
coreInst.removeAddress(address)
try:
if (int(coreInst._utils.getEpoch()) - int(coreInst.getPeerInfo(address, 'dateSeen'))) >= 600:
expireTime = 600
else:
expireTime = 86400
coreInst._blacklist.addToDB(address, dataType=1, expire=expireTime)
except sqlite3.IntegrityError: #TODO just make sure its not a unique constraint issue
pass
except ValueError:
pass
logger.warn('Removed address ' + address + '.')
minScore = int(config.get('peers.minimum_score', -100))
maxPeers = int(config.get('peers.max_stored', 5000))
for address in adders:
# Remove peers that go below the negative score
if PeerProfiles(address, coreInst).score < minScore:
coreInst.removeAddress(address)
try:
if (int(coreInst._utils.getEpoch()) - int(coreInst.getPeerInfo(address, 'dateSeen'))) >= 600:
expireTime = 600
else:
expireTime = 86400
coreInst._blacklist.addToDB(address, dataType=1, expire=expireTime)
except sqlite3.IntegrityError: #TODO just make sure its not a unique constraint issue
pass
except ValueError:
pass
logger.warn('Removed address ' + address + '.')
# Unban probably not malicious peers TODO improve
coreInst._blacklist.deleteExpired(dataType=1)

19
onionr/onionrutils.py
View File

@ -159,16 +159,19 @@ class OnionrUtils:
config.reload()
self.getTimeBypassToken()
# TODO: URL encode parameters, just as an extra measure. May not be needed, but should be added regardless.
try:
with open(self._core.dataDir + 'host.txt', 'r') as host:
hostname = host.read()
except FileNotFoundError:
return False
payload = 'http://%s:%s/client/?action=%s&token=%s&timingToken=%s' % (hostname, config.get('client.port'), command, config.get('client.webpassword'), self.timingToken)
hostname = ''
while hostname == '':
try:
with open(self._core.privateApiHostFile, 'r') as host:
hostname = host.read()
except FileNotFoundError:
print('wat')
time.sleep(1)
if data != '':
payload += '&data=' + urllib.parse.quote_plus(data)
data = '&data=' + urllib.parse.quote_plus(data)
payload = 'http://%s:%s/%s%s' % (hostname, config.get('client.client.port'), command, data)
try:
retData = requests.get(payload).text
retData = requests.get(payload, headers={'token': config.get('client.webpassword')}).text
except Exception as error:
if not silent:
logger.error('Failed to make local request (command: %s):%s' % (command, error))

3
onionr/static-data/default_config.json
View File

@ -7,7 +7,8 @@
"socket_servers": false,
"security_level": 0,
"max_block_age": 2678400,
"public_key": ""
"public_key": "",
"use_new_api_server": false
},
"www" : {

4
onionr/static-data/index.html
View File

@ -1,7 +1,7 @@
<h1>This is an Onionr Node</h1>
<p>The content on this server is not necessarily created by the server owner, and was not necessarily stored specifically with the owner's knowledge of its contents.</p>
<p>The content on this server was not necessarily intentionally stored or created by the owner(s) of this server.</p>
<p>Onionr is a decentralized data storage system that anyone can insert data into.</p>
<p>Onionr is a decentralized peer-to-peer data storage system.</p>
<p>To learn more about Onionr, see the website at <a href="https://onionr.voidnet.tech/">https://Onionr.VoidNet.tech/</a></p>

1
requirements.txt
View File

@ -2,7 +2,6 @@ urllib3==1.23
requests==2.20.0
PyNaCl==1.2.1
gevent==1.3.6
sha3==0.2.1
defusedxml==0.5.0
Flask==1.0.2
PySocks==1.6.8