+ added reverse block insertion
* handle downloading of blocks better when peer goes offline * bumped default disk allocation * added post request util
This commit is contained in:
Kevin Froman
parent
8e1b6e1e7e
commit
71007a2d0a
@ -24,7 +24,7 @@ from gevent.wsgi import WSGIServer
|
||||
import sys, random, threading, hmac, hashlib, base64, time, math, os, logger, config
|
||||
from core import Core
|
||||
from onionrblockapi import Block
|
||||
import onionrutils, onionrcrypto
|
||||
import onionrutils, onionrcrypto, blockimporter
|
||||
|
||||
class API:
|
||||
'''
|
||||
@ -141,9 +141,6 @@ class API:
|
||||
resp = Response('Goodbye')
|
||||
elif action == 'ping':
|
||||
resp = Response('pong')
|
||||
elif action == 'stats':
|
||||
resp = Response('me_irl')
|
||||
raise Exception
|
||||
elif action == 'site':
|
||||
block = data
|
||||
siteData = self._core.getData(data)
|
||||
@ -175,6 +172,24 @@ class API:
|
||||
resp = Response("")
|
||||
return resp
|
||||
|
||||
@app.route('/public/upload/', methods=['POST'])
|
||||
def blockUpload():
|
||||
self.validateHost('public')
|
||||
resp = 'failure'
|
||||
try:
|
||||
data = request.form['block']
|
||||
except KeyError:
|
||||
logger.warn('No block specified for upload')
|
||||
pass
|
||||
else:
|
||||
if sys.getsizeof(data) < 100000000:
|
||||
if blockimporter.importBlockFromData(data, self._core):
|
||||
resp = 'success'
|
||||
else:
|
||||
logger.warn('Error encountered importing uploaded block')
|
||||
|
||||
resp = Response(resp)
|
||||
return resp
|
||||
@app.route('/public/')
|
||||
def public_handler():
|
||||
# Public means it is publicly network accessible
|
||||
@ -198,7 +213,6 @@ class API:
|
||||
resp = Response('\n'.join(self._core.getBlockList()))
|
||||
elif action == 'directMessage':
|
||||
resp = Response(self._core.handle_direct_connection(data))
|
||||
#elif action == 'nodeProof':
|
||||
|
||||
elif action == 'announce':
|
||||
if data != '':
|
||||
|
||||
@ -37,6 +37,8 @@ class OnionrCommunicatorDaemon:
|
||||
self.nistSaltTimestamp = 0
|
||||
self.powSalt = 0
|
||||
|
||||
self.blockToUpload = ''
|
||||
|
||||
# loop time.sleep delay in seconds
|
||||
self.delay = 1
|
||||
|
||||
@ -111,7 +113,6 @@ class OnionrCommunicatorDaemon:
|
||||
peer = self.pickOnlinePeer()
|
||||
newKeys = self.peerAction(peer, action='kex')
|
||||
self._core._utils.mergeKeys(newKeys)
|
||||
|
||||
self.decrementThreadCount('lookupKeys')
|
||||
return
|
||||
|
||||
@ -339,10 +340,31 @@ class OnionrCommunicatorDaemon:
|
||||
for i in self.timers:
|
||||
if i.timerFunction.__name__ == 'lookupKeys':
|
||||
i.count = (i.frequency - 1)
|
||||
elif cmd[0] == 'uploadBlock':
|
||||
self.blockToUpload = cmd[1]
|
||||
threading.Thread(target=self.uploadBlock).start()
|
||||
else:
|
||||
logger.info('Recieved daemonQueue command:' + cmd[0])
|
||||
self.decrementThreadCount('daemonCommands')
|
||||
|
||||
def uploadBlock(self):
|
||||
tiredPeers = []
|
||||
if not self._core._utils.validateHash(self.blockToUpload):
|
||||
logger.warn('Requested to upload invalid block')
|
||||
return
|
||||
for i in max(len(self.onlinePeers), 2):
|
||||
while True:
|
||||
peer = self.pickOnlinePeer()
|
||||
if peer
|
||||
url = 'http://' + peer + '/public/upload/'
|
||||
data = {'block': block.Block(self.blockToUpload).getRaw()}
|
||||
if peer.endswith('.onion'):
|
||||
proxyType = 'tor'
|
||||
elif peer.endswith('.i2p'):
|
||||
proxyType = 'i2p'
|
||||
logger.info("Uploading block")
|
||||
self._core._utils.doPostRequest(url, data=data, proxyType=proxyType)
|
||||
|
||||
def announce(self, peer):
|
||||
'''Announce to peers our address'''
|
||||
announceCount = 0
|
||||
|
||||
@ -41,10 +41,12 @@ class Core:
|
||||
self.blockDataLocation = 'data/blocks/'
|
||||
self.addressDB = 'data/address.db'
|
||||
self.hsAdder = ''
|
||||
|
||||
self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt'
|
||||
self.bootstrapList = []
|
||||
self.requirements = onionrvalues.OnionrValues()
|
||||
self.torPort = torPort
|
||||
|
||||
self.usageFile = 'data/disk-usage.txt'
|
||||
|
||||
if not os.path.exists('data/'):
|
||||
os.mkdir('data/')
|
||||
@ -757,6 +759,7 @@ class Core:
|
||||
retData = self.setData(payload)
|
||||
self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
|
||||
self.setBlockType(retData, meta['type'])
|
||||
self.daemonQueueAdd('uploadBlock', retData)
|
||||
|
||||
if retData != False:
|
||||
events.event('insertBlock', onionr = None, threaded = False)
|
||||
|
||||
@ -199,7 +199,7 @@ class Onionr:
|
||||
'connect': self.addAddress,
|
||||
'kex': self.doKEX,
|
||||
|
||||
'getpassword': self.getWebPassword
|
||||
'getpassword': self.printWebPassword
|
||||
}
|
||||
|
||||
self.cmdhelp = {
|
||||
@ -259,6 +259,9 @@ class Onionr:
|
||||
def getWebPassword(self):
|
||||
return config.get('client.hmac')
|
||||
|
||||
def printWebPassword(self):
|
||||
print(self.getWebPassword())
|
||||
|
||||
def getHelp(self):
|
||||
return self.cmdhelp
|
||||
|
||||
|
||||
@ -42,6 +42,10 @@ class InvalidHexHash(Exception):
|
||||
'''When a string is not a valid hex string of appropriate length for a hash value'''
|
||||
pass
|
||||
|
||||
class InvalidProof(Exception):
|
||||
'''When a proof is invalid or inadequate'''
|
||||
pass
|
||||
|
||||
# network level exceptions
|
||||
class MissingPort(Exception):
|
||||
pass
|
||||
|
||||
@ -54,21 +54,12 @@ class OnionrUtils:
|
||||
except Exception as error:
|
||||
logger.error('Failed to fetch time bypass token.', error=error)
|
||||
|
||||
def sendPM(self, pubkey, message):
|
||||
def getRoundedEpoch(self, roundS=60):
|
||||
'''
|
||||
High level function to encrypt a message to a peer and insert it as a block
|
||||
'''
|
||||
|
||||
self._core.insertBlock(message, header='pm', sign=True, encryptType='asym', asymPeer=pubkey)
|
||||
|
||||
return
|
||||
|
||||
def getCurrentHourEpoch(self):
|
||||
'''
|
||||
Returns the current epoch, rounded down to the hour
|
||||
Returns the epoch, rounded down to given seconds (Default 60)
|
||||
'''
|
||||
epoch = self.getEpoch()
|
||||
return epoch - (epoch % 3600)
|
||||
return epoch - (epoch % roundS)
|
||||
|
||||
def incrementAddressSuccess(self, address):
|
||||
'''
|
||||
@ -134,6 +125,7 @@ class OnionrUtils:
|
||||
if newAdderList != False:
|
||||
for adder in newAdderList.split(','):
|
||||
if not adder in self._core.listAdders(randomOrder = False) and adder.strip() != self.getMyAddress():
|
||||
if adder[:4] == '0000':
|
||||
if self._core.addAddress(adder):
|
||||
logger.info('Added %s to db.' % adder, timestamp = True)
|
||||
retVal = True
|
||||
@ -210,11 +202,18 @@ class OnionrUtils:
|
||||
|
||||
'''
|
||||
meta = {}
|
||||
metadata = {}
|
||||
data = blockData
|
||||
try:
|
||||
blockData = blockData.encode()
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
try:
|
||||
metadata = json.loads(blockData[:blockData.find(b'\n')].decode())
|
||||
except json.decoder.JSONDecodeError:
|
||||
pass
|
||||
else:
|
||||
data = blockData[blockData.find(b'\n'):].decode()
|
||||
|
||||
if not metadata['encryptType'] in ('asym', 'sym'):
|
||||
@ -525,6 +524,30 @@ class OnionrUtils:
|
||||
'''returns epoch'''
|
||||
return math.floor(time.time())
|
||||
|
||||
def doPostRequest(self, url, data={}, port=0, proxyType='tor'):
|
||||
'''
|
||||
Do a POST request through a local tor or i2p instance
|
||||
'''
|
||||
if proxyType == 'tor':
|
||||
if port == 0:
|
||||
port = self._core.torPort
|
||||
proxies = {'http': 'socks5://127.0.0.1:' + str(port), 'https': 'socks5://127.0.0.1:' + str(port)}
|
||||
elif proxyType == 'i2p':
|
||||
proxies = {'http': 'http://127.0.0.1:4444'}
|
||||
else:
|
||||
return
|
||||
headers = {'user-agent': 'PyOnionr'}
|
||||
try:
|
||||
proxies = {'http': 'socks5h://127.0.0.1:' + str(port), 'https': 'socks5h://127.0.0.1:' + str(port)}
|
||||
r = requests.post(url, data=data, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30))
|
||||
retData = r.text
|
||||
except KeyboardInterrupt:
|
||||
raise KeyboardInterrupt
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.debug('Error: %s' % str(e))
|
||||
retData = False
|
||||
return retData
|
||||
|
||||
def doGetRequest(self, url, port=0, proxyType='tor'):
|
||||
'''
|
||||
Do a get request through a local tor or i2p instance
|
||||
@ -549,7 +572,7 @@ class OnionrUtils:
|
||||
retData = False
|
||||
return retData
|
||||
|
||||
def getNistBeaconSalt(self, torPort=0):
|
||||
def getNistBeaconSalt(self, torPort=0, rounding=3600):
|
||||
'''
|
||||
Get the token for the current hour from the NIST randomness beacon
|
||||
'''
|
||||
@ -559,7 +582,7 @@ class OnionrUtils:
|
||||
except IndexError:
|
||||
raise onionrexceptions.MissingPort('Missing Tor socks port')
|
||||
retData = ''
|
||||
curTime = self._core._utils.getCurrentHourEpoch
|
||||
curTime = self.getRoundedEpoch(rounding)
|
||||
self.nistSaltTimestamp = curTime
|
||||
data = self.doGetRequest('https://beacon.nist.gov/rest/record/' + str(curTime), port=torPort)
|
||||
dataXML = minidom.parseString(data, forbid_dtd=True, forbid_entities=True, forbid_external=True)
|
||||
|
||||
@ -33,7 +33,7 @@
|
||||
},
|
||||
|
||||
"allocations":{
|
||||
"disk": 1000000000,
|
||||
"disk": 9000000000,
|
||||
"netTotal": 1000000000,
|
||||
"blockCache" : 5000000,
|
||||
"blockCacheTotal" : 50000000
|
||||
|
||||
Loading…
Reference in New Issue
Block a user