kev/Onionr
1
0
Fork 0
Code Issues 24 Pull Requests Projects Releases 2 Wiki Activity

+ Reformatted API, more efficient, standard, and secure now

Browse Source 
* Various bug fixes
This commit is contained in:
Kevin Froman 2018-12-22 13:02:09 -06:00
parent 53f98c3449
commit 1dd471b91e
No known key found for this signature in database
GPG Key ID: 0D414D0FE405B63B
11 changed files with 174 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
onionr/api.py
View File

@ -25,8 +25,6 @@ import core
from onionrblockapi import Block
import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config, onionr
API_VERSION = 0
def guessMime(path):
'''
Guesses the mime type of a file from the input filename
@ -67,6 +65,7 @@ class PublicAPI:
self.torAdder = clientAPI._core.hsAddress
self.i2pAdder = clientAPI._core.i2pAddress
self.bindPort = config.get('client.public.port')
logger.info('Running public api on %s:%s' % (self.host, self.bindPort))
@app.before_request
def validateRequest():
@ -84,7 +83,7 @@ class PublicAPI:
resp.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'"
resp.headers['X-Frame-Options'] = 'deny'
resp.headers['X-Content-Type-Options'] = "nosniff"
resp.headers['X-API'] = API_VERSION
resp.headers['X-API'] = onionr.API_VERSION
return resp
@app.route('/')
@ -105,8 +104,9 @@ class PublicAPI:
return Response('\n'.join(bList))
@app.route('/getdata/<name>')
def getBlockData():
def getBlockData(name):
resp = ''
data = name
if clientAPI._utils.validateHash(data):
if data not in self.hideBlocks:
if os.path.exists(clientAPI._core.dataDir + 'blocks/' + data + '.dat'):
@ -117,20 +117,64 @@ class PublicAPI:
resp = ""
return Response(resp)
@app.route('/www/<path:path>')
def wwwPublic(path):
if not config.get("www.public.run", True):
abort(403)
return send_from_directory(config.get('www.public.path', 'static-data/www/public/'), path)
@app.route('/ping')
def ping():
return Response("pong!")
@app.route('/getdbhash')
def getDBHash():
return Response(clientAPI._utils.getBlockDBHash())
@app.route('/pex')
def peerExchange():
response = ','.join(self._core.listAdders())
response = ','.join(clientAPI._core.listAdders())
if len(response) == 0:
response = 'none'
resp = Response(response)
return Response(response)
@app.route('/announce', methods=['post'])
def acceptAnnounce():
resp = 'failure'
powHash = ''
randomData = ''
newNode = ''
ourAdder = clientAPI._core.hsAddress.encode()
try:
newNode = request.form['node'].encode()
except KeyError:
logger.warn('No block specified for upload')
pass
else:
try:
randomData = request.form['random']
randomData = base64.b64decode(randomData)
except KeyError:
logger.warn('No random data specified for upload')
else:
nodes = newNode + clientAPI._core.hsAddress.encode()
nodes = clientAPI._core._crypto.blake2bHash(nodes)
powHash = clientAPI._core._crypto.blake2bHash(randomData + nodes)
try:
powHash = powHash.decode()
except AttributeError:
pass
if powHash.startswith('0000'):
try:
newNode = newNode.decode()
except AttributeError:
pass
if clientAPI._core.addAddress(newNode):
resp = 'Success'
else:
logger.warn(newNode.decode() + ' failed to meet POW: ' + powHash)
resp = Response(resp)
return resp
@app.route('/upload', methods=['post'])
def upload():
@ -156,6 +200,10 @@ class PublicAPI:
return resp
clientAPI.setPublicAPIInstance(self)
while self.torAdder == '':
clientAPI._core.refreshFirstStartVars()
self.torAdder = clientAPI._core.hsAddress
time.sleep(1)
self.httpServer = WSGIServer((self.host, self.bindPort), app, log=None)
self.httpServer.serve_forever()
@ -166,7 +214,7 @@ class API:
callbacks = {'public' : {}, 'private' : {}}
def __init__(self, debug, API_VERSION):
def __init__(self, onionrInst, debug, API_VERSION):
'''
Initialize the api server, preping variables for later use
@ -190,10 +238,11 @@ class API:
self.timeBypassToken = base64.b16encode(os.urandom(32)).decode()
self.publicAPI = None # gets set when the thread calls our setter... bad hack but kinda necessary with flask
threading.Thread(target=PublicAPI, args=(self,)).start()
#threading.Thread(target=PublicAPI, args=(self,)).start()
self.host = setBindIP(self._core.privateApiHostFile)
logger.info('Running api on %s:%s' % (self.host, self.bindPort))
self.httpServer = ''
onionrInst.setClientAPIInst(self)
@app.before_request
def validateRequest():
@ -205,20 +254,20 @@ class API:
abort(403)
except KeyError:
abort(403)
@app.after_request
def afterReq(resp):
resp.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'"
resp.headers['X-Frame-Options'] = 'deny'
resp.headers['X-Content-Type-Options'] = "nosniff"
resp.headers['X-API'] = API_VERSION
resp.headers['Server'] = 'nginx'
resp.headers['X-API'] = onionr.API_VERSION
resp.headers['Server'] = ''
resp.headers['Date'] = 'Thu, 1 Jan 1970 00:00:00 GMT' # Clock info is probably useful to attackers. Set to unix epoch.
return resp
@app.route('/ping')
def ping():
return Respose("pong!")
return Response("pong!")
@app.route('/')
def hello():
@ -256,10 +305,10 @@ class API:
except AttributeError:
pass
return Response("bye")
self.httpServer = WSGIServer((self.host, bindPort), app, log=None)
self.httpServer.serve_forever()
def setPublicAPIInstance(self, inst):
assert isinstance(inst, PublicAPI)
self.publicAPI = inst
@ -277,4 +326,4 @@ class API:
else:
return True
except TypeError:
return False
return False

15
onionr/communicator2.py
View File

@ -19,11 +19,10 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
#import gevent.monkey
#gevent.monkey.patch_all()
import sys, os, core, config, json, requests, time, logger, threading, base64, onionr, uuid
import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block
import onionrdaemontools, onionrsockets, onionrchat, onionr, onionrproofs
import binascii
from dependencies import secrets
from defusedxml import minidom
@ -103,6 +102,7 @@ class OnionrCommunicatorDaemon:
OnionrCommunicatorTimers(self, self.daemonTools.cooldownPeer, 30, requiresPeer=True)
OnionrCommunicatorTimers(self, self.uploadBlock, 10, requiresPeer=True, maxThreads=1)
OnionrCommunicatorTimers(self, self.daemonCommands, 6, maxThreads=1)
OnionrCommunicatorTimers(self, self.detectAPICrash, 5, maxThreads=1)
deniableBlockTimer = OnionrCommunicatorTimers(self, self.daemonTools.insertDeniableBlock, 180, requiresPeer=True, maxThreads=1)
netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600)
@ -232,7 +232,10 @@ class OnionrCommunicatorDaemon:
content = content.encode()
except AttributeError:
pass
content = base64.b64decode(content) # content is base64 encoded in transport
try:
content = base64.b64decode(content) # content is base64 encoded in transport
except binascii.Error:
pass
realHash = self._core._crypto.sha3Hash(content)
try:
realHash = realHash.decode() # bytes on some versions for some reason
@ -423,7 +426,7 @@ class OnionrCommunicatorDaemon:
if len(peer) == 0:
return False
#logger.debug('Performing ' + action + ' with ' + peer + ' on port ' + str(self.proxyPort))
url = 'http://' + peer + '/' + action
url = 'http://%s/%s' % (peer, action)
if len(data) > 0:
url += '&data=' + data
@ -546,9 +549,9 @@ class OnionrCommunicatorDaemon:
def detectAPICrash(self):
'''exit if the api server crashes/stops'''
if self._core._utils.localCommand('ping', silent=False) != 'pong':
if self._core._utils.localCommand('ping', silent=False) not in ('pong', 'pong!'):
for i in range(5):
if self._core._utils.localCommand('ping') == 'pong':
if self._core._utils.localCommand('ping') in ('pong', 'pong!'):
break # break for loop
time.sleep(1)
else:

2
onionr/core.py
View File

@ -154,7 +154,7 @@ class Core:
if address == config.get('i2p.ownAddr', None) or address == self.hsAddress:
return False
if type(address) is type(None) or len(address) == 0:
if type(address) is None or len(address) == 0:
return False
if self._utils.validateID(address):
conn = sqlite3.connect(self.addressDB, timeout=10)

2
onionr/netcontroller.py
View File

@ -77,7 +77,7 @@ class NetController:
hsVer = '# v2 onions'
if config.get('tor.v3onions'):
hsVer = 'HiddenServiceVersion 3'
logger.debug('Using v3 onions :)')
logger.debug('Using v3 onions')
if os.path.exists(self.torConfigLocation):
os.remove(self.torConfigLocation)

112
onionr/onionr.py
View File

@ -67,6 +67,7 @@ class Onionr:
data_exists = Onionr.setupConfig(self.dataDir, self = self)
self.onionrCore = core.Core()
#self.deleteRunFiles()
self.onionrUtils = onionrutils.OnionrUtils(self.onionrCore)
self.clientAPIInst = '' # Client http api instance
@ -399,6 +400,16 @@ class Onionr:
logger.info('Syntax: friend add/remove/list [address]')
def deleteRunFiles(self):
try:
os.remove(self.onionrCore.publicApiHostFile)
except FileNotFoundError:
pass
try:
os.remove(self.onionrCore.privateApiHostFile)
except FileNotFoundError:
pass
def banBlock(self):
try:
ban = sys.argv[2]
@ -695,6 +706,13 @@ class Onionr:
os.remove('.onionr-lock')
except FileNotFoundError:
pass
def setClientAPIInst(self, inst):
self.clientAPIInst = inst
def getClientApi(self):
while self.clientAPIInst == '':
time.sleep(0.5)
return self.clientAPIInst
def daemon(self):
'''
@ -708,64 +726,66 @@ class Onionr:
logger.debug('Runcheck file found on daemon start, deleting in advance.')
os.remove('data/.runcheck')
apiTarget = api.API
apiThread = Thread(target = apiTarget, args = (self.debug, API_VERSION))
apiThread.start()
Thread(target=api.API, args=(self, self.debug, API_VERSION)).start()
Thread(target=api.PublicAPI, args=[self.getClientApi()]).start()
try:
time.sleep(3)
time.sleep(0)
except KeyboardInterrupt:
logger.debug('Got keyboard interrupt, shutting down...')
time.sleep(1)
self.onionrUtils.localCommand('shutdown')
apiHost = ''
while apiHost == '':
try:
with open(self.onionrCore.publicApiHostFile, 'r') as hostFile:
apiHost = hostFile.read()
except FileNotFoundError:
pass
time.sleep(0.5)
Onionr.setupConfig('data/', self = self)
if self._developmentMode:
logger.warn('DEVELOPMENT MODE ENABLED (LESS SECURE)', timestamp = False)
net = NetController(config.get('client.public.port', 59497), apiServerIP=apiHost)
logger.debug('Tor is starting...')
if not net.startTor():
self.onionrUtils.localCommand('shutdown')
sys.exit(1)
if len(net.myID) > 0 and config.get('general.security_level') == 0:
logger.debug('Started .onion service: %s' % (logger.colors.underline + net.myID))
else:
apiHost = '127.0.0.1'
if apiThread.isAlive():
try:
with open(self.onionrCore.publicApiHostFile, 'r') as hostFile:
apiHost = hostFile.read()
except FileNotFoundError:
pass
Onionr.setupConfig('data/', self = self)
logger.debug('.onion service disabled')
logger.debug('Using public key: %s' % (logger.colors.underline + self.onionrCore._crypto.pubKey))
time.sleep(1)
if self._developmentMode:
logger.warn('DEVELOPMENT MODE ENABLED (LESS SECURE)', timestamp = False)
net = NetController(config.get('client.public.port', 59497), apiServerIP=apiHost)
logger.debug('Tor is starting...')
if not net.startTor():
self.onionrUtils.localCommand('shutdown')
sys.exit(1)
if len(net.myID) > 0 and config.get('general.security_level') == 0:
logger.debug('Started .onion service: %s' % (logger.colors.underline + net.myID))
else:
logger.debug('.onion service disabled')
logger.debug('Using public key: %s' % (logger.colors.underline + self.onionrCore._crypto.pubKey))
time.sleep(1)
# TODO: make runable on windows
communicatorProc = subprocess.Popen([communicatorDaemon, 'run', str(net.socksPort)])
# TODO: make runable on windows
communicatorProc = subprocess.Popen([communicatorDaemon, 'run', str(net.socksPort)])
# print nice header thing :)
if config.get('general.display_header', True):
self.header()
# print nice header thing :)
if config.get('general.display_header', True):
self.header()
# print out debug info
self.version(verbosity = 5, function = logger.debug)
logger.debug('Python version %s' % platform.python_version())
# print out debug info
self.version(verbosity = 5, function = logger.debug)
logger.debug('Python version %s' % platform.python_version())
logger.debug('Started communicator.')
logger.debug('Started communicator.')
events.event('daemon_start', onionr = self)
try:
while True:
time.sleep(5)
events.event('daemon_start', onionr = self)
try:
while True:
time.sleep(5)
# Break if communicator process ends, so we don't have left over processes
if communicatorProc.poll() is not None:
break
except KeyboardInterrupt:
self.onionrCore.daemonQueueAdd('shutdown')
self.onionrUtils.localCommand('shutdown')
# Break if communicator process ends, so we don't have left over processes
if communicatorProc.poll() is not None:
break
except KeyboardInterrupt:
self.onionrCore.daemonQueueAdd('shutdown')
self.onionrUtils.localCommand('shutdown')
time.sleep(3)
self.deleteRunFiles()
net.killTor()
return
def killDaemon(self):

2
onionr/onionrdaemontools.py
View File

@ -45,7 +45,7 @@ class DaemonTools:
ourID = self.daemon._core.hsAddress.strip()
url = 'http://' + peer + '/public/announce/'
url = 'http://' + peer + '/announce'
data = {'node': ourID}
combinedNodes = ourID + peer

38
onionr/onionrpeers.py
View File

@ -79,27 +79,29 @@ def peerCleanup(coreInst):
logger.info('Cleaning peers...')
config.reload()
minScore = int(config.get('peers.minimum_score', -100))
maxPeers = int(config.get('peers.max_stored', 5000))
adders = getScoreSortedPeerList(coreInst)
adders.reverse()
if len(adders) > 1:
for address in adders:
# Remove peers that go below the negative score
if PeerProfiles(address, coreInst).score < minScore:
coreInst.removeAddress(address)
try:
if (int(coreInst._utils.getEpoch()) - int(coreInst.getPeerInfo(address, 'dateSeen'))) >= 600:
expireTime = 600
else:
expireTime = 86400
coreInst._blacklist.addToDB(address, dataType=1, expire=expireTime)
except sqlite3.IntegrityError: #TODO just make sure its not a unique constraint issue
pass
except ValueError:
pass
logger.warn('Removed address ' + address + '.')
minScore = int(config.get('peers.minimum_score', -100))
maxPeers = int(config.get('peers.max_stored', 5000))
for address in adders:
# Remove peers that go below the negative score
if PeerProfiles(address, coreInst).score < minScore:
coreInst.removeAddress(address)
try:
if (int(coreInst._utils.getEpoch()) - int(coreInst.getPeerInfo(address, 'dateSeen'))) >= 600:
expireTime = 600
else:
expireTime = 86400
coreInst._blacklist.addToDB(address, dataType=1, expire=expireTime)
except sqlite3.IntegrityError: #TODO just make sure its not a unique constraint issue
pass
except ValueError:
pass
logger.warn('Removed address ' + address + '.')
# Unban probably not malicious peers TODO improve
coreInst._blacklist.deleteExpired(dataType=1)

1
onionr/onionrproofs.py
View File

@ -238,7 +238,6 @@ class POW:
break
else:
time.sleep(2)
print('boi')
except KeyboardInterrupt:
self.shutdown()
logger.warn('Got keyboard interrupt while waiting for POW result, stopping')

16
onionr/onionrutils.py
View File

@ -159,17 +159,17 @@ class OnionrUtils:
config.reload()
self.getTimeBypassToken()
# TODO: URL encode parameters, just as an extra measure. May not be needed, but should be added regardless.
try:
with open(self._core.privateApiHostFile, 'r') as host:
hostname = host.read()
except FileNotFoundError:
return False
hostname = ''
while hostname == '':
try:
with open(self._core.privateApiHostFile, 'r') as host:
hostname = host.read()
except FileNotFoundError:
print('wat')
time.sleep(1)
if data != '':
data = '&data=' + urllib.parse.quote_plus(data)
payload = 'http://%s:%s/%s%s' % (hostname, config.get('client.client.port'), command, data)
#payload = 'http://%s:%s/client/?action=%s&token=%s&timingToken=%s' % (hostname, config.get('client.client.port'), command, config.get('client.webpassword'), self.timingToken)
#if data != '':
# payload += '&data=' + urllib.parse.quote_plus(data)
try:
retData = requests.get(payload, headers={'token': config.get('client.webpassword')}).text
except Exception as error:

4
onionr/static-data/index.html
View File

@ -1,7 +1,7 @@
<h1>This is an Onionr Node</h1>
<p>The content on this server is not necessarily created by the server owner, and was not necessarily stored specifically with the owner's knowledge of its contents.</p>
<p>The content on this server was not necessarily intentionally stored or created by the owner(s) of this server.</p>
<p>Onionr is a decentralized data storage system that anyone can insert data into.</p>
<p>Onionr is a decentralized peer-to-peer data storage system.</p>
<p>To learn more about Onionr, see the website at <a href="https://onionr.voidnet.tech/">https://Onionr.VoidNet.tech/</a></p>

1
requirements.txt
View File

@ -2,7 +2,6 @@ urllib3==1.23
requests==2.20.0
PyNaCl==1.2.1
gevent==1.3.6
sha3==0.2.1
defusedxml==0.5.0
Flask==1.0.2
PySocks==1.6.8