+ added reverse block insertion

* handle downloading of blocks better when peer goes offline
* bumped default disk allocation
* added post request util
This commit is contained in:
Kevin Froman 2018-07-23 02:43:10 -05:00
parent 8e1b6e1e7e
commit 71007a2d0a
No known key found for this signature in database
GPG Key ID: 0D414D0FE405B63B
7 changed files with 106 additions and 37 deletions

View File

@ -24,7 +24,7 @@ from gevent.wsgi import WSGIServer
import sys, random, threading, hmac, hashlib, base64, time, math, os, logger, config
from core import Core
from onionrblockapi import Block
import onionrutils, onionrcrypto
import onionrutils, onionrcrypto, blockimporter
class API:
'''
@ -141,9 +141,6 @@ class API:
resp = Response('Goodbye')
elif action == 'ping':
resp = Response('pong')
elif action == 'stats':
resp = Response('me_irl')
raise Exception
elif action == 'site':
block = data
siteData = self._core.getData(data)
@ -175,6 +172,24 @@ class API:
resp = Response("")
return resp
@app.route('/public/upload/', methods=['POST'])
def blockUpload():
self.validateHost('public')
resp = 'failure'
try:
data = request.form['block']
except KeyError:
logger.warn('No block specified for upload')
pass
else:
if sys.getsizeof(data) < 100000000:
if blockimporter.importBlockFromData(data, self._core):
resp = 'success'
else:
logger.warn('Error encountered importing uploaded block')
resp = Response(resp)
return resp
@app.route('/public/')
def public_handler():
# Public means it is publicly network accessible
@ -198,7 +213,6 @@ class API:
resp = Response('\n'.join(self._core.getBlockList()))
elif action == 'directMessage':
resp = Response(self._core.handle_direct_connection(data))
#elif action == 'nodeProof':
elif action == 'announce':
if data != '':

View File

@ -37,6 +37,8 @@ class OnionrCommunicatorDaemon:
self.nistSaltTimestamp = 0
self.powSalt = 0
self.blockToUpload = ''
# loop time.sleep delay in seconds
self.delay = 1
@ -111,7 +113,6 @@ class OnionrCommunicatorDaemon:
peer = self.pickOnlinePeer()
newKeys = self.peerAction(peer, action='kex')
self._core._utils.mergeKeys(newKeys)
self.decrementThreadCount('lookupKeys')
return
@ -339,10 +340,31 @@ class OnionrCommunicatorDaemon:
for i in self.timers:
if i.timerFunction.__name__ == 'lookupKeys':
i.count = (i.frequency - 1)
elif cmd[0] == 'uploadBlock':
self.blockToUpload = cmd[1]
threading.Thread(target=self.uploadBlock).start()
else:
logger.info('Recieved daemonQueue command:' + cmd[0])
self.decrementThreadCount('daemonCommands')
def uploadBlock(self):
tiredPeers = []
if not self._core._utils.validateHash(self.blockToUpload):
logger.warn('Requested to upload invalid block')
return
for i in max(len(self.onlinePeers), 2):
while True:
peer = self.pickOnlinePeer()
if peer
url = 'http://' + peer + '/public/upload/'
data = {'block': block.Block(self.blockToUpload).getRaw()}
if peer.endswith('.onion'):
proxyType = 'tor'
elif peer.endswith('.i2p'):
proxyType = 'i2p'
logger.info("Uploading block")
self._core._utils.doPostRequest(url, data=data, proxyType=proxyType)
def announce(self, peer):
'''Announce to peers our address'''
announceCount = 0

View File

@ -41,10 +41,12 @@ class Core:
self.blockDataLocation = 'data/blocks/'
self.addressDB = 'data/address.db'
self.hsAdder = ''
self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt'
self.bootstrapList = []
self.requirements = onionrvalues.OnionrValues()
self.torPort = torPort
self.usageFile = 'data/disk-usage.txt'
if not os.path.exists('data/'):
os.mkdir('data/')
@ -757,6 +759,7 @@ class Core:
retData = self.setData(payload)
self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
self.setBlockType(retData, meta['type'])
self.daemonQueueAdd('uploadBlock', retData)
if retData != False:
events.event('insertBlock', onionr = None, threaded = False)

View File

@ -199,7 +199,7 @@ class Onionr:
'connect': self.addAddress,
'kex': self.doKEX,
'getpassword': self.getWebPassword
'getpassword': self.printWebPassword
}
self.cmdhelp = {
@ -259,6 +259,9 @@ class Onionr:
def getWebPassword(self):
return config.get('client.hmac')
def printWebPassword(self):
print(self.getWebPassword())
def getHelp(self):
return self.cmdhelp

View File

@ -42,6 +42,10 @@ class InvalidHexHash(Exception):
'''When a string is not a valid hex string of appropriate length for a hash value'''
pass
class InvalidProof(Exception):
'''When a proof is invalid or inadequate'''
pass
# network level exceptions
class MissingPort(Exception):
pass

View File

@ -54,21 +54,12 @@ class OnionrUtils:
except Exception as error:
logger.error('Failed to fetch time bypass token.', error=error)
def sendPM(self, pubkey, message):
def getRoundedEpoch(self, roundS=60):
'''
High level function to encrypt a message to a peer and insert it as a block
'''
self._core.insertBlock(message, header='pm', sign=True, encryptType='asym', asymPeer=pubkey)
return
def getCurrentHourEpoch(self):
'''
Returns the current epoch, rounded down to the hour
Returns the epoch, rounded down to given seconds (Default 60)
'''
epoch = self.getEpoch()
return epoch - (epoch % 3600)
return epoch - (epoch % roundS)
def incrementAddressSuccess(self, address):
'''
@ -134,6 +125,7 @@ class OnionrUtils:
if newAdderList != False:
for adder in newAdderList.split(','):
if not adder in self._core.listAdders(randomOrder = False) and adder.strip() != self.getMyAddress():
if adder[:4] == '0000':
if self._core.addAddress(adder):
logger.info('Added %s to db.' % adder, timestamp = True)
retVal = True
@ -210,11 +202,18 @@ class OnionrUtils:
'''
meta = {}
metadata = {}
data = blockData
try:
blockData = blockData.encode()
except AttributeError:
pass
try:
metadata = json.loads(blockData[:blockData.find(b'\n')].decode())
except json.decoder.JSONDecodeError:
pass
else:
data = blockData[blockData.find(b'\n'):].decode()
if not metadata['encryptType'] in ('asym', 'sym'):
@ -525,6 +524,30 @@ class OnionrUtils:
'''returns epoch'''
return math.floor(time.time())
def doPostRequest(self, url, data={}, port=0, proxyType='tor'):
'''
Do a POST request through a local tor or i2p instance
'''
if proxyType == 'tor':
if port == 0:
port = self._core.torPort
proxies = {'http': 'socks5://127.0.0.1:' + str(port), 'https': 'socks5://127.0.0.1:' + str(port)}
elif proxyType == 'i2p':
proxies = {'http': 'http://127.0.0.1:4444'}
else:
return
headers = {'user-agent': 'PyOnionr'}
try:
proxies = {'http': 'socks5h://127.0.0.1:' + str(port), 'https': 'socks5h://127.0.0.1:' + str(port)}
r = requests.post(url, data=data, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30))
retData = r.text
except KeyboardInterrupt:
raise KeyboardInterrupt
except requests.exceptions.RequestException as e:
logger.debug('Error: %s' % str(e))
retData = False
return retData
def doGetRequest(self, url, port=0, proxyType='tor'):
'''
Do a get request through a local tor or i2p instance
@ -549,7 +572,7 @@ class OnionrUtils:
retData = False
return retData
def getNistBeaconSalt(self, torPort=0):
def getNistBeaconSalt(self, torPort=0, rounding=3600):
'''
Get the token for the current hour from the NIST randomness beacon
'''
@ -559,7 +582,7 @@ class OnionrUtils:
except IndexError:
raise onionrexceptions.MissingPort('Missing Tor socks port')
retData = ''
curTime = self._core._utils.getCurrentHourEpoch
curTime = self.getRoundedEpoch(rounding)
self.nistSaltTimestamp = curTime
data = self.doGetRequest('https://beacon.nist.gov/rest/record/' + str(curTime), port=torPort)
dataXML = minidom.parseString(data, forbid_dtd=True, forbid_entities=True, forbid_external=True)

View File

@ -33,7 +33,7 @@
},
"allocations":{
"disk": 1000000000,
"disk": 9000000000,
"netTotal": 1000000000,
"blockCache" : 5000000,
"blockCacheTotal" : 50000000