kev/Onionr
1
0
Fork 0
Code Issues 24 Pull Requests Projects Releases 2 Wiki Activity

+ added reverse block insertion

Browse Source 
* handle downloading of blocks better when peer goes offline
* bumped default disk allocation
* added post request util
This commit is contained in:
Kevin Froman 2018-07-23 02:43:10 -05:00
parent 8e1b6e1e7e
commit 71007a2d0a
No known key found for this signature in database
GPG Key ID: 0D414D0FE405B63B
7 changed files with 106 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
onionr/api.py
View File

@ -24,7 +24,7 @@ from gevent.wsgi import WSGIServer
import sys, random, threading, hmac, hashlib, base64, time, math, os, logger, config
from core import Core
from onionrblockapi import Block
import onionrutils, onionrcrypto
import onionrutils, onionrcrypto, blockimporter
class API:
'''
@ -141,9 +141,6 @@ class API:
resp = Response('Goodbye')
elif action == 'ping':
resp = Response('pong')
elif action == 'stats':
resp = Response('me_irl')
raise Exception
elif action == 'site':
block = data
siteData = self._core.getData(data)
@ -175,6 +172,24 @@ class API:
resp = Response("")
return resp
@app.route('/public/upload/', methods=['POST'])
def blockUpload():
self.validateHost('public')
resp = 'failure'
try:
data = request.form['block']
except KeyError:
logger.warn('No block specified for upload')
pass
else:
if sys.getsizeof(data) < 100000000:
if blockimporter.importBlockFromData(data, self._core):
resp = 'success'
else:
logger.warn('Error encountered importing uploaded block')
resp = Response(resp)
return resp
@app.route('/public/')
def public_handler():
# Public means it is publicly network accessible
@ -198,7 +213,6 @@ class API:
resp = Response('\n'.join(self._core.getBlockList()))
elif action == 'directMessage':
resp = Response(self._core.handle_direct_connection(data))
#elif action == 'nodeProof':
elif action == 'announce':
if data != '':

30
onionr/communicator2.py
View File

@ -36,6 +36,8 @@ class OnionrCommunicatorDaemon:
# intalize NIST beacon salt and time
self.nistSaltTimestamp = 0
self.powSalt = 0
self.blockToUpload = ''
# loop time.sleep delay in seconds
self.delay = 1
@ -84,7 +86,7 @@ class OnionrCommunicatorDaemon:
OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True)
# set loop to execute instantly to load up peer pool (replaced old pool init wait)
peerPoolTimer.count = (peerPoolTimer.frequency - 1)
peerPoolTimer.count = (peerPoolTimer.frequency - 1)
# Main daemon loop, mainly for calling timers, don't do any complex operations here to avoid locking
try:
@ -101,7 +103,7 @@ class OnionrCommunicatorDaemon:
logger.info('Goodbye.')
self._core._utils.localCommand('shutdown')
time.sleep(0.5)
def lookupKeys(self):
'''Lookup new keys'''
logger.debug('Looking up new keys...')
@ -111,7 +113,6 @@ class OnionrCommunicatorDaemon:
peer = self.pickOnlinePeer()
newKeys = self.peerAction(peer, action='kex')
self._core._utils.mergeKeys(newKeys)
self.decrementThreadCount('lookupKeys')
return
@ -196,7 +197,7 @@ class OnionrCommunicatorDaemon:
pass
logger.warn('Block hash validation failed for ' + blockHash + ' got ' + tempHash)
self.blockQueue.remove(blockHash) # remove from block queue both if success or false
self.currentDownloading.remove(blockHash)
self.currentDownloading.remove(blockHash)
self.decrementThreadCount('getBlocks')
return
@ -339,10 +340,31 @@ class OnionrCommunicatorDaemon:
for i in self.timers:
if i.timerFunction.__name__ == 'lookupKeys':
i.count = (i.frequency - 1)
elif cmd[0] == 'uploadBlock':
self.blockToUpload = cmd[1]
threading.Thread(target=self.uploadBlock).start()
else:
logger.info('Recieved daemonQueue command:' + cmd[0])
self.decrementThreadCount('daemonCommands')
def uploadBlock(self):
tiredPeers = []
if not self._core._utils.validateHash(self.blockToUpload):
logger.warn('Requested to upload invalid block')
return
for i in max(len(self.onlinePeers), 2):
while True:
peer = self.pickOnlinePeer()
if peer
url = 'http://' + peer + '/public/upload/'
data = {'block': block.Block(self.blockToUpload).getRaw()}
if peer.endswith('.onion'):
proxyType = 'tor'
elif peer.endswith('.i2p'):
proxyType = 'i2p'
logger.info("Uploading block")
self._core._utils.doPostRequest(url, data=data, proxyType=proxyType)
def announce(self, peer):
'''Announce to peers our address'''
announceCount = 0

5
onionr/core.py
View File

@ -41,10 +41,12 @@ class Core:
self.blockDataLocation = 'data/blocks/'
self.addressDB = 'data/address.db'
self.hsAdder = ''
self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt'
self.bootstrapList = []
self.requirements = onionrvalues.OnionrValues()
self.torPort = torPort
self.usageFile = 'data/disk-usage.txt'
if not os.path.exists('data/'):
os.mkdir('data/')
@ -757,6 +759,7 @@ class Core:
retData = self.setData(payload)
self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
self.setBlockType(retData, meta['type'])
self.daemonQueueAdd('uploadBlock', retData)
if retData != False:
events.event('insertBlock', onionr = None, threaded = False)

5
onionr/onionr.py
View File

@ -199,7 +199,7 @@ class Onionr:
'connect': self.addAddress,
'kex': self.doKEX,
'getpassword': self.getWebPassword
'getpassword': self.printWebPassword
}
self.cmdhelp = {
@ -258,6 +258,9 @@ class Onionr:
def getWebPassword(self):
return config.get('client.hmac')
def printWebPassword(self):
print(self.getWebPassword())
def getHelp(self):
return self.cmdhelp

4
onionr/onionrexceptions.py
View File

@ -42,6 +42,10 @@ class InvalidHexHash(Exception):
'''When a string is not a valid hex string of appropriate length for a hash value'''
pass
class InvalidProof(Exception):
'''When a proof is invalid or inadequate'''
pass
# network level exceptions
class MissingPort(Exception):
pass

73
onionr/onionrutils.py
View File

@ -54,21 +54,12 @@ class OnionrUtils:
except Exception as error:
logger.error('Failed to fetch time bypass token.', error=error)
def sendPM(self, pubkey, message):
def getRoundedEpoch(self, roundS=60):
'''
High level function to encrypt a message to a peer and insert it as a block
'''
self._core.insertBlock(message, header='pm', sign=True, encryptType='asym', asymPeer=pubkey)
return
def getCurrentHourEpoch(self):
'''
Returns the current epoch, rounded down to the hour
Returns the epoch, rounded down to given seconds (Default 60)
'''
epoch = self.getEpoch()
return epoch - (epoch % 3600)
return epoch - (epoch % roundS)
def incrementAddressSuccess(self, address):
'''
@ -134,9 +125,10 @@ class OnionrUtils:
if newAdderList != False:
for adder in newAdderList.split(','):
if not adder in self._core.listAdders(randomOrder = False) and adder.strip() != self.getMyAddress():
if self._core.addAddress(adder):
logger.info('Added %s to db.' % adder, timestamp = True)
retVal = True
if adder[:4] == '0000':
if self._core.addAddress(adder):
logger.info('Added %s to db.' % adder, timestamp = True)
retVal = True
else:
pass
#logger.debug('%s is either our address or already in our DB' % adder)
@ -210,19 +202,26 @@ class OnionrUtils:
'''
meta = {}
metadata = {}
data = blockData
try:
blockData = blockData.encode()
except AttributeError:
pass
metadata = json.loads(blockData[:blockData.find(b'\n')].decode())
data = blockData[blockData.find(b'\n'):].decode()
try:
metadata = json.loads(blockData[:blockData.find(b'\n')].decode())
except json.decoder.JSONDecodeError:
pass
else:
data = blockData[blockData.find(b'\n'):].decode()
if not metadata['encryptType'] in ('asym', 'sym'):
try:
meta = json.loads(metadata['meta'])
except KeyError:
pass
meta = metadata['meta']
if not metadata['encryptType'] in ('asym', 'sym'):
try:
meta = json.loads(metadata['meta'])
except KeyError:
pass
meta = metadata['meta']
return (metadata, meta, data)
def checkPort(self, port, host=''):
@ -525,6 +524,30 @@ class OnionrUtils:
'''returns epoch'''
return math.floor(time.time())
def doPostRequest(self, url, data={}, port=0, proxyType='tor'):
'''
Do a POST request through a local tor or i2p instance
'''
if proxyType == 'tor':
if port == 0:
port = self._core.torPort
proxies = {'http': 'socks5://127.0.0.1:' + str(port), 'https': 'socks5://127.0.0.1:' + str(port)}
elif proxyType == 'i2p':
proxies = {'http': 'http://127.0.0.1:4444'}
else:
return
headers = {'user-agent': 'PyOnionr'}
try:
proxies = {'http': 'socks5h://127.0.0.1:' + str(port), 'https': 'socks5h://127.0.0.1:' + str(port)}
r = requests.post(url, data=data, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30))
retData = r.text
except KeyboardInterrupt:
raise KeyboardInterrupt
except requests.exceptions.RequestException as e:
logger.debug('Error: %s' % str(e))
retData = False
return retData
def doGetRequest(self, url, port=0, proxyType='tor'):
'''
Do a get request through a local tor or i2p instance
@ -549,7 +572,7 @@ class OnionrUtils:
retData = False
return retData
def getNistBeaconSalt(self, torPort=0):
def getNistBeaconSalt(self, torPort=0, rounding=3600):
'''
Get the token for the current hour from the NIST randomness beacon
'''
@ -559,7 +582,7 @@ class OnionrUtils:
except IndexError:
raise onionrexceptions.MissingPort('Missing Tor socks port')
retData = ''
curTime = self._core._utils.getCurrentHourEpoch
curTime = self.getRoundedEpoch(rounding)
self.nistSaltTimestamp = curTime
data = self.doGetRequest('https://beacon.nist.gov/rest/record/' + str(curTime), port=torPort)
dataXML = minidom.parseString(data, forbid_dtd=True, forbid_entities=True, forbid_external=True)

2
onionr/static-data/default_config.json
View File

@ -33,7 +33,7 @@
},
"allocations":{
"disk": 1000000000,
"disk": 9000000000,
"netTotal": 1000000000,
"blockCache" : 5000000,
"blockCacheTotal" : 50000000