+ added reverse block insertion

* handle downloading of blocks better when peer goes offline
* bumped default disk allocation
* added post request util
This commit is contained in:
Kevin Froman 2018-07-23 02:43:10 -05:00
parent 8e1b6e1e7e
commit 71007a2d0a
No known key found for this signature in database
GPG Key ID: 0D414D0FE405B63B
7 changed files with 106 additions and 37 deletions

View File

@ -24,7 +24,7 @@ from gevent.wsgi import WSGIServer
import sys, random, threading, hmac, hashlib, base64, time, math, os, logger, config import sys, random, threading, hmac, hashlib, base64, time, math, os, logger, config
from core import Core from core import Core
from onionrblockapi import Block from onionrblockapi import Block
import onionrutils, onionrcrypto import onionrutils, onionrcrypto, blockimporter
class API: class API:
''' '''
@ -141,9 +141,6 @@ class API:
resp = Response('Goodbye') resp = Response('Goodbye')
elif action == 'ping': elif action == 'ping':
resp = Response('pong') resp = Response('pong')
elif action == 'stats':
resp = Response('me_irl')
raise Exception
elif action == 'site': elif action == 'site':
block = data block = data
siteData = self._core.getData(data) siteData = self._core.getData(data)
@ -175,6 +172,24 @@ class API:
resp = Response("") resp = Response("")
return resp return resp
@app.route('/public/upload/', methods=['POST'])
def blockUpload():
self.validateHost('public')
resp = 'failure'
try:
data = request.form['block']
except KeyError:
logger.warn('No block specified for upload')
pass
else:
if sys.getsizeof(data) < 100000000:
if blockimporter.importBlockFromData(data, self._core):
resp = 'success'
else:
logger.warn('Error encountered importing uploaded block')
resp = Response(resp)
return resp
@app.route('/public/') @app.route('/public/')
def public_handler(): def public_handler():
# Public means it is publicly network accessible # Public means it is publicly network accessible
@ -198,7 +213,6 @@ class API:
resp = Response('\n'.join(self._core.getBlockList())) resp = Response('\n'.join(self._core.getBlockList()))
elif action == 'directMessage': elif action == 'directMessage':
resp = Response(self._core.handle_direct_connection(data)) resp = Response(self._core.handle_direct_connection(data))
#elif action == 'nodeProof':
elif action == 'announce': elif action == 'announce':
if data != '': if data != '':

View File

@ -37,6 +37,8 @@ class OnionrCommunicatorDaemon:
self.nistSaltTimestamp = 0 self.nistSaltTimestamp = 0
self.powSalt = 0 self.powSalt = 0
self.blockToUpload = ''
# loop time.sleep delay in seconds # loop time.sleep delay in seconds
self.delay = 1 self.delay = 1
@ -111,7 +113,6 @@ class OnionrCommunicatorDaemon:
peer = self.pickOnlinePeer() peer = self.pickOnlinePeer()
newKeys = self.peerAction(peer, action='kex') newKeys = self.peerAction(peer, action='kex')
self._core._utils.mergeKeys(newKeys) self._core._utils.mergeKeys(newKeys)
self.decrementThreadCount('lookupKeys') self.decrementThreadCount('lookupKeys')
return return
@ -339,10 +340,31 @@ class OnionrCommunicatorDaemon:
for i in self.timers: for i in self.timers:
if i.timerFunction.__name__ == 'lookupKeys': if i.timerFunction.__name__ == 'lookupKeys':
i.count = (i.frequency - 1) i.count = (i.frequency - 1)
elif cmd[0] == 'uploadBlock':
self.blockToUpload = cmd[1]
threading.Thread(target=self.uploadBlock).start()
else: else:
logger.info('Recieved daemonQueue command:' + cmd[0]) logger.info('Recieved daemonQueue command:' + cmd[0])
self.decrementThreadCount('daemonCommands') self.decrementThreadCount('daemonCommands')
def uploadBlock(self):
tiredPeers = []
if not self._core._utils.validateHash(self.blockToUpload):
logger.warn('Requested to upload invalid block')
return
for i in max(len(self.onlinePeers), 2):
while True:
peer = self.pickOnlinePeer()
if peer
url = 'http://' + peer + '/public/upload/'
data = {'block': block.Block(self.blockToUpload).getRaw()}
if peer.endswith('.onion'):
proxyType = 'tor'
elif peer.endswith('.i2p'):
proxyType = 'i2p'
logger.info("Uploading block")
self._core._utils.doPostRequest(url, data=data, proxyType=proxyType)
def announce(self, peer): def announce(self, peer):
'''Announce to peers our address''' '''Announce to peers our address'''
announceCount = 0 announceCount = 0

View File

@ -41,10 +41,12 @@ class Core:
self.blockDataLocation = 'data/blocks/' self.blockDataLocation = 'data/blocks/'
self.addressDB = 'data/address.db' self.addressDB = 'data/address.db'
self.hsAdder = '' self.hsAdder = ''
self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt' self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt'
self.bootstrapList = [] self.bootstrapList = []
self.requirements = onionrvalues.OnionrValues() self.requirements = onionrvalues.OnionrValues()
self.torPort = torPort
self.usageFile = 'data/disk-usage.txt'
if not os.path.exists('data/'): if not os.path.exists('data/'):
os.mkdir('data/') os.mkdir('data/')
@ -757,6 +759,7 @@ class Core:
retData = self.setData(payload) retData = self.setData(payload)
self.addToBlockDB(retData, selfInsert=True, dataSaved=True) self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
self.setBlockType(retData, meta['type']) self.setBlockType(retData, meta['type'])
self.daemonQueueAdd('uploadBlock', retData)
if retData != False: if retData != False:
events.event('insertBlock', onionr = None, threaded = False) events.event('insertBlock', onionr = None, threaded = False)

View File

@ -199,7 +199,7 @@ class Onionr:
'connect': self.addAddress, 'connect': self.addAddress,
'kex': self.doKEX, 'kex': self.doKEX,
'getpassword': self.getWebPassword 'getpassword': self.printWebPassword
} }
self.cmdhelp = { self.cmdhelp = {
@ -259,6 +259,9 @@ class Onionr:
def getWebPassword(self): def getWebPassword(self):
return config.get('client.hmac') return config.get('client.hmac')
def printWebPassword(self):
print(self.getWebPassword())
def getHelp(self): def getHelp(self):
return self.cmdhelp return self.cmdhelp

View File

@ -42,6 +42,10 @@ class InvalidHexHash(Exception):
'''When a string is not a valid hex string of appropriate length for a hash value''' '''When a string is not a valid hex string of appropriate length for a hash value'''
pass pass
class InvalidProof(Exception):
'''When a proof is invalid or inadequate'''
pass
# network level exceptions # network level exceptions
class MissingPort(Exception): class MissingPort(Exception):
pass pass

View File

@ -54,21 +54,12 @@ class OnionrUtils:
except Exception as error: except Exception as error:
logger.error('Failed to fetch time bypass token.', error=error) logger.error('Failed to fetch time bypass token.', error=error)
def sendPM(self, pubkey, message): def getRoundedEpoch(self, roundS=60):
''' '''
High level function to encrypt a message to a peer and insert it as a block Returns the epoch, rounded down to given seconds (Default 60)
'''
self._core.insertBlock(message, header='pm', sign=True, encryptType='asym', asymPeer=pubkey)
return
def getCurrentHourEpoch(self):
'''
Returns the current epoch, rounded down to the hour
''' '''
epoch = self.getEpoch() epoch = self.getEpoch()
return epoch - (epoch % 3600) return epoch - (epoch % roundS)
def incrementAddressSuccess(self, address): def incrementAddressSuccess(self, address):
''' '''
@ -134,6 +125,7 @@ class OnionrUtils:
if newAdderList != False: if newAdderList != False:
for adder in newAdderList.split(','): for adder in newAdderList.split(','):
if not adder in self._core.listAdders(randomOrder = False) and adder.strip() != self.getMyAddress(): if not adder in self._core.listAdders(randomOrder = False) and adder.strip() != self.getMyAddress():
if adder[:4] == '0000':
if self._core.addAddress(adder): if self._core.addAddress(adder):
logger.info('Added %s to db.' % adder, timestamp = True) logger.info('Added %s to db.' % adder, timestamp = True)
retVal = True retVal = True
@ -210,11 +202,18 @@ class OnionrUtils:
''' '''
meta = {} meta = {}
metadata = {}
data = blockData
try: try:
blockData = blockData.encode() blockData = blockData.encode()
except AttributeError: except AttributeError:
pass pass
try:
metadata = json.loads(blockData[:blockData.find(b'\n')].decode()) metadata = json.loads(blockData[:blockData.find(b'\n')].decode())
except json.decoder.JSONDecodeError:
pass
else:
data = blockData[blockData.find(b'\n'):].decode() data = blockData[blockData.find(b'\n'):].decode()
if not metadata['encryptType'] in ('asym', 'sym'): if not metadata['encryptType'] in ('asym', 'sym'):
@ -525,6 +524,30 @@ class OnionrUtils:
'''returns epoch''' '''returns epoch'''
return math.floor(time.time()) return math.floor(time.time())
def doPostRequest(self, url, data={}, port=0, proxyType='tor'):
'''
Do a POST request through a local tor or i2p instance
'''
if proxyType == 'tor':
if port == 0:
port = self._core.torPort
proxies = {'http': 'socks5://127.0.0.1:' + str(port), 'https': 'socks5://127.0.0.1:' + str(port)}
elif proxyType == 'i2p':
proxies = {'http': 'http://127.0.0.1:4444'}
else:
return
headers = {'user-agent': 'PyOnionr'}
try:
proxies = {'http': 'socks5h://127.0.0.1:' + str(port), 'https': 'socks5h://127.0.0.1:' + str(port)}
r = requests.post(url, data=data, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30))
retData = r.text
except KeyboardInterrupt:
raise KeyboardInterrupt
except requests.exceptions.RequestException as e:
logger.debug('Error: %s' % str(e))
retData = False
return retData
def doGetRequest(self, url, port=0, proxyType='tor'): def doGetRequest(self, url, port=0, proxyType='tor'):
''' '''
Do a get request through a local tor or i2p instance Do a get request through a local tor or i2p instance
@ -549,7 +572,7 @@ class OnionrUtils:
retData = False retData = False
return retData return retData
def getNistBeaconSalt(self, torPort=0): def getNistBeaconSalt(self, torPort=0, rounding=3600):
''' '''
Get the token for the current hour from the NIST randomness beacon Get the token for the current hour from the NIST randomness beacon
''' '''
@ -559,7 +582,7 @@ class OnionrUtils:
except IndexError: except IndexError:
raise onionrexceptions.MissingPort('Missing Tor socks port') raise onionrexceptions.MissingPort('Missing Tor socks port')
retData = '' retData = ''
curTime = self._core._utils.getCurrentHourEpoch curTime = self.getRoundedEpoch(rounding)
self.nistSaltTimestamp = curTime self.nistSaltTimestamp = curTime
data = self.doGetRequest('https://beacon.nist.gov/rest/record/' + str(curTime), port=torPort) data = self.doGetRequest('https://beacon.nist.gov/rest/record/' + str(curTime), port=torPort)
dataXML = minidom.parseString(data, forbid_dtd=True, forbid_entities=True, forbid_external=True) dataXML = minidom.parseString(data, forbid_dtd=True, forbid_entities=True, forbid_external=True)

View File

@ -33,7 +33,7 @@
}, },
"allocations":{ "allocations":{
"disk": 1000000000, "disk": 9000000000,
"netTotal": 1000000000, "netTotal": 1000000000,
"blockCache" : 5000000, "blockCache" : 5000000,
"blockCacheTotal" : 50000000 "blockCacheTotal" : 50000000