Merge branch 'node-pow' into 'master'
Reverse block insertion and other bug fixes See merge request beardog/Onionr!4
This commit is contained in:
Kevin
commit
83c4dbcb72
|
|
@ -24,7 +24,7 @@ from gevent.wsgi import WSGIServer
|
|||
import sys, random, threading, hmac, hashlib, base64, time, math, os, logger, config
|
||||
from core import Core
|
||||
from onionrblockapi import Block
|
||||
import onionrutils, onionrcrypto
|
||||
import onionrutils, onionrcrypto, blockimporter
|
||||
|
||||
class API:
|
||||
'''
|
||||
|
|
@ -141,9 +141,6 @@ class API:
|
|||
resp = Response('Goodbye')
|
||||
elif action == 'ping':
|
||||
resp = Response('pong')
|
||||
elif action == 'stats':
|
||||
resp = Response('me_irl')
|
||||
raise Exception
|
||||
elif action == 'site':
|
||||
block = data
|
||||
siteData = self._core.getData(data)
|
||||
|
|
@ -175,6 +172,24 @@ class API:
|
|||
resp = Response("")
|
||||
return resp
|
||||
|
||||
@app.route('/public/upload/', methods=['POST'])
|
||||
def blockUpload():
|
||||
self.validateHost('public')
|
||||
resp = 'failure'
|
||||
try:
|
||||
data = request.form['block']
|
||||
except KeyError:
|
||||
logger.warn('No block specified for upload')
|
||||
pass
|
||||
else:
|
||||
if sys.getsizeof(data) < 100000000:
|
||||
if blockimporter.importBlockFromData(data, self._core):
|
||||
resp = 'success'
|
||||
else:
|
||||
logger.warn('Error encountered importing uploaded block')
|
||||
|
||||
resp = Response(resp)
|
||||
return resp
|
||||
@app.route('/public/')
|
||||
def public_handler():
|
||||
# Public means it is publicly network accessible
|
||||
|
|
@ -198,6 +213,7 @@ class API:
|
|||
resp = Response('\n'.join(self._core.getBlockList()))
|
||||
elif action == 'directMessage':
|
||||
resp = Response(self._core.handle_direct_connection(data))
|
||||
|
||||
elif action == 'announce':
|
||||
if data != '':
|
||||
# TODO: require POW for this
|
||||
|
|
|
|||
|
|
@ -0,0 +1,40 @@
|
|||
'''
|
||||
Onionr - P2P Microblogging Platform & Social network
|
||||
|
||||
Import block data and save it
|
||||
'''
|
||||
'''
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
'''
|
||||
import core, onionrexceptions, logger
|
||||
def importBlockFromData(content, coreInst):
|
||||
retData = False
|
||||
if not isinstance(coreInst, core.Core):
|
||||
raise Exception("coreInst must be an Onionr core instance")
|
||||
|
||||
try:
|
||||
content = content.encode()
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
metas = coreInst._utils.getBlockMetadataFromData(content) # returns tuple(metadata, meta), meta is also in metadata
|
||||
metadata = metas[0]
|
||||
if coreInst._utils.validateMetadata(metadata): # check if metadata is valid
|
||||
if coreInst._crypto.verifyPow(content): # check if POW is enough/correct
|
||||
logger.info('Block passed proof, saving.')
|
||||
blockHash = coreInst.setData(content)
|
||||
blockHash = coreInst.addToBlockDB(blockHash, dataSaved=True)
|
||||
coreInst._utils.processBlockMetadata(blockHash) # caches block metadata values to block database
|
||||
retData = True
|
||||
return retData
|
||||
|
|
@ -36,6 +36,8 @@ class OnionrCommunicatorDaemon:
|
|||
# intalize NIST beacon salt and time
|
||||
self.nistSaltTimestamp = 0
|
||||
self.powSalt = 0
|
||||
|
||||
self.blockToUpload = ''
|
||||
|
||||
# loop time.sleep delay in seconds
|
||||
self.delay = 1
|
||||
|
|
@ -84,7 +86,7 @@ class OnionrCommunicatorDaemon:
|
|||
OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True)
|
||||
|
||||
# set loop to execute instantly to load up peer pool (replaced old pool init wait)
|
||||
peerPoolTimer.count = (peerPoolTimer.frequency - 1)
|
||||
peerPoolTimer.count = (peerPoolTimer.frequency - 1)
|
||||
|
||||
# Main daemon loop, mainly for calling timers, don't do any complex operations here to avoid locking
|
||||
try:
|
||||
|
|
@ -101,7 +103,7 @@ class OnionrCommunicatorDaemon:
|
|||
logger.info('Goodbye.')
|
||||
self._core._utils.localCommand('shutdown')
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def lookupKeys(self):
|
||||
'''Lookup new keys'''
|
||||
logger.debug('Looking up new keys...')
|
||||
|
|
@ -111,7 +113,6 @@ class OnionrCommunicatorDaemon:
|
|||
peer = self.pickOnlinePeer()
|
||||
newKeys = self.peerAction(peer, action='kex')
|
||||
self._core._utils.mergeKeys(newKeys)
|
||||
|
||||
self.decrementThreadCount('lookupKeys')
|
||||
return
|
||||
|
||||
|
|
@ -196,7 +197,7 @@ class OnionrCommunicatorDaemon:
|
|||
pass
|
||||
logger.warn('Block hash validation failed for ' + blockHash + ' got ' + tempHash)
|
||||
self.blockQueue.remove(blockHash) # remove from block queue both if success or false
|
||||
self.currentDownloading.remove(blockHash)
|
||||
self.currentDownloading.remove(blockHash)
|
||||
self.decrementThreadCount('getBlocks')
|
||||
return
|
||||
|
||||
|
|
@ -339,10 +340,32 @@ class OnionrCommunicatorDaemon:
|
|||
for i in self.timers:
|
||||
if i.timerFunction.__name__ == 'lookupKeys':
|
||||
i.count = (i.frequency - 1)
|
||||
elif cmd[0] == 'uploadBlock':
|
||||
self.blockToUpload = cmd[1]
|
||||
threading.Thread(target=self.uploadBlock).start()
|
||||
else:
|
||||
logger.info('Recieved daemonQueue command:' + cmd[0])
|
||||
self.decrementThreadCount('daemonCommands')
|
||||
|
||||
def uploadBlock(self):
|
||||
triedPeers = []
|
||||
if not self._core._utils.validateHash(self.blockToUpload):
|
||||
logger.warn('Requested to upload invalid block')
|
||||
return
|
||||
for i in range(max(len(self.onlinePeers), 2)):
|
||||
peer = self.pickOnlinePeer()
|
||||
if peer in triedPeers:
|
||||
continue
|
||||
triedPeers.append(peer)
|
||||
url = 'http://' + peer + '/public/upload/'
|
||||
data = {'block': block.Block(self.blockToUpload).getRaw()}
|
||||
if peer.endswith('.onion'):
|
||||
proxyType = 'tor'
|
||||
elif peer.endswith('.i2p'):
|
||||
proxyType = 'i2p'
|
||||
logger.info("Uploading block")
|
||||
self._core._utils.doPostRequest(url, data=data, proxyType=proxyType)
|
||||
|
||||
def announce(self, peer):
|
||||
'''Announce to peers our address'''
|
||||
announceCount = 0
|
||||
|
|
|
|||
|
|
@ -41,10 +41,12 @@ class Core:
|
|||
self.blockDataLocation = 'data/blocks/'
|
||||
self.addressDB = 'data/address.db'
|
||||
self.hsAdder = ''
|
||||
|
||||
self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt'
|
||||
self.bootstrapList = []
|
||||
self.requirements = onionrvalues.OnionrValues()
|
||||
self.torPort = torPort
|
||||
|
||||
self.usageFile = 'data/disk-usage.txt'
|
||||
|
||||
if not os.path.exists('data/'):
|
||||
os.mkdir('data/')
|
||||
|
|
@ -578,25 +580,6 @@ class Core:
|
|||
conn.close()
|
||||
return
|
||||
|
||||
def handle_direct_connection(self, data):
|
||||
'''
|
||||
Handles direct messages
|
||||
'''
|
||||
try:
|
||||
data = json.loads(data)
|
||||
|
||||
# TODO: Determine the sender, verify, etc
|
||||
if ('callback' in data) and (data['callback'] is True):
|
||||
# then this is a response to the message we sent earlier
|
||||
self.daemonQueueAdd('checkCallbacks', json.dumps(data))
|
||||
else:
|
||||
# then we should handle it and respond accordingly
|
||||
self.daemonQueueAdd('incomingDirectConnection', json.dumps(data))
|
||||
except Exception as e:
|
||||
logger.warn('Failed to handle incoming direct message: %s' % str(e))
|
||||
|
||||
return
|
||||
|
||||
def getBlockList(self, unsaved = False): # TODO: Use unsaved??
|
||||
'''
|
||||
Get list of our blocks
|
||||
|
|
@ -757,6 +740,7 @@ class Core:
|
|||
retData = self.setData(payload)
|
||||
self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
|
||||
self.setBlockType(retData, meta['type'])
|
||||
self.daemonQueueAdd('uploadBlock', retData)
|
||||
|
||||
if retData != False:
|
||||
events.event('insertBlock', onionr = None, threaded = False)
|
||||
|
|
|
|||
|
|
@ -199,7 +199,7 @@ class Onionr:
|
|||
'connect': self.addAddress,
|
||||
'kex': self.doKEX,
|
||||
|
||||
'getpassword': self.getWebPassword
|
||||
'getpassword': self.printWebPassword
|
||||
}
|
||||
|
||||
self.cmdhelp = {
|
||||
|
|
@ -258,6 +258,9 @@ class Onionr:
|
|||
|
||||
def getWebPassword(self):
|
||||
return config.get('client.hmac')
|
||||
|
||||
def printWebPassword(self):
|
||||
print(self.getWebPassword())
|
||||
|
||||
def getHelp(self):
|
||||
return self.cmdhelp
|
||||
|
|
|
|||
|
|
@ -42,6 +42,10 @@ class InvalidHexHash(Exception):
|
|||
'''When a string is not a valid hex string of appropriate length for a hash value'''
|
||||
pass
|
||||
|
||||
class InvalidProof(Exception):
|
||||
'''When a proof is invalid or inadequate'''
|
||||
pass
|
||||
|
||||
# network level exceptions
|
||||
class MissingPort(Exception):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -22,16 +22,19 @@ import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, lo
|
|||
import core
|
||||
|
||||
class DataPOW:
|
||||
def __init__(self, data, threadCount = 5):
|
||||
def __init__(self, data, forceDifficulty=0, threadCount = 5):
|
||||
self.foundHash = False
|
||||
self.difficulty = 0
|
||||
self.data = data
|
||||
self.threadCount = threadCount
|
||||
|
||||
dataLen = sys.getsizeof(data)
|
||||
self.difficulty = math.floor(dataLen / 1000000)
|
||||
if self.difficulty <= 2:
|
||||
self.difficulty = 4
|
||||
if forceDifficulty == 0:
|
||||
dataLen = sys.getsizeof(data)
|
||||
self.difficulty = math.floor(dataLen / 1000000)
|
||||
if self.difficulty <= 2:
|
||||
self.difficulty = 4
|
||||
else:
|
||||
self.difficulty = forceDifficulty
|
||||
|
||||
try:
|
||||
self.data = self.data.encode()
|
||||
|
|
|
|||
|
|
@ -54,21 +54,12 @@ class OnionrUtils:
|
|||
except Exception as error:
|
||||
logger.error('Failed to fetch time bypass token.', error=error)
|
||||
|
||||
def sendPM(self, pubkey, message):
|
||||
def getRoundedEpoch(self, roundS=60):
|
||||
'''
|
||||
High level function to encrypt a message to a peer and insert it as a block
|
||||
'''
|
||||
|
||||
self._core.insertBlock(message, header='pm', sign=True, encryptType='asym', asymPeer=pubkey)
|
||||
|
||||
return
|
||||
|
||||
def getCurrentHourEpoch(self):
|
||||
'''
|
||||
Returns the current epoch, rounded down to the hour
|
||||
Returns the epoch, rounded down to given seconds (Default 60)
|
||||
'''
|
||||
epoch = self.getEpoch()
|
||||
return epoch - (epoch % 3600)
|
||||
return epoch - (epoch % roundS)
|
||||
|
||||
def incrementAddressSuccess(self, address):
|
||||
'''
|
||||
|
|
@ -134,9 +125,10 @@ class OnionrUtils:
|
|||
if newAdderList != False:
|
||||
for adder in newAdderList.split(','):
|
||||
if not adder in self._core.listAdders(randomOrder = False) and adder.strip() != self.getMyAddress():
|
||||
if self._core.addAddress(adder):
|
||||
logger.info('Added %s to db.' % adder, timestamp = True)
|
||||
retVal = True
|
||||
if adder[:4] == '0000':
|
||||
if self._core.addAddress(adder):
|
||||
logger.info('Added %s to db.' % adder, timestamp = True)
|
||||
retVal = True
|
||||
else:
|
||||
pass
|
||||
#logger.debug('%s is either our address or already in our DB' % adder)
|
||||
|
|
@ -210,19 +202,26 @@ class OnionrUtils:
|
|||
|
||||
'''
|
||||
meta = {}
|
||||
metadata = {}
|
||||
data = blockData
|
||||
try:
|
||||
blockData = blockData.encode()
|
||||
except AttributeError:
|
||||
pass
|
||||
metadata = json.loads(blockData[:blockData.find(b'\n')].decode())
|
||||
data = blockData[blockData.find(b'\n'):].decode()
|
||||
|
||||
try:
|
||||
metadata = json.loads(blockData[:blockData.find(b'\n')].decode())
|
||||
except json.decoder.JSONDecodeError:
|
||||
pass
|
||||
else:
|
||||
data = blockData[blockData.find(b'\n'):].decode()
|
||||
|
||||
if not metadata['encryptType'] in ('asym', 'sym'):
|
||||
try:
|
||||
meta = json.loads(metadata['meta'])
|
||||
except KeyError:
|
||||
pass
|
||||
meta = metadata['meta']
|
||||
if not metadata['encryptType'] in ('asym', 'sym'):
|
||||
try:
|
||||
meta = json.loads(metadata['meta'])
|
||||
except KeyError:
|
||||
pass
|
||||
meta = metadata['meta']
|
||||
return (metadata, meta, data)
|
||||
|
||||
def checkPort(self, port, host=''):
|
||||
|
|
@ -525,6 +524,30 @@ class OnionrUtils:
|
|||
'''returns epoch'''
|
||||
return math.floor(time.time())
|
||||
|
||||
def doPostRequest(self, url, data={}, port=0, proxyType='tor'):
|
||||
'''
|
||||
Do a POST request through a local tor or i2p instance
|
||||
'''
|
||||
if proxyType == 'tor':
|
||||
if port == 0:
|
||||
port = self._core.torPort
|
||||
proxies = {'http': 'socks5://127.0.0.1:' + str(port), 'https': 'socks5://127.0.0.1:' + str(port)}
|
||||
elif proxyType == 'i2p':
|
||||
proxies = {'http': 'http://127.0.0.1:4444'}
|
||||
else:
|
||||
return
|
||||
headers = {'user-agent': 'PyOnionr'}
|
||||
try:
|
||||
proxies = {'http': 'socks5h://127.0.0.1:' + str(port), 'https': 'socks5h://127.0.0.1:' + str(port)}
|
||||
r = requests.post(url, data=data, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30))
|
||||
retData = r.text
|
||||
except KeyboardInterrupt:
|
||||
raise KeyboardInterrupt
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.debug('Error: %s' % str(e))
|
||||
retData = False
|
||||
return retData
|
||||
|
||||
def doGetRequest(self, url, port=0, proxyType='tor'):
|
||||
'''
|
||||
Do a get request through a local tor or i2p instance
|
||||
|
|
@ -549,7 +572,7 @@ class OnionrUtils:
|
|||
retData = False
|
||||
return retData
|
||||
|
||||
def getNistBeaconSalt(self, torPort=0):
|
||||
def getNistBeaconSalt(self, torPort=0, rounding=3600):
|
||||
'''
|
||||
Get the token for the current hour from the NIST randomness beacon
|
||||
'''
|
||||
|
|
@ -559,7 +582,7 @@ class OnionrUtils:
|
|||
except IndexError:
|
||||
raise onionrexceptions.MissingPort('Missing Tor socks port')
|
||||
retData = ''
|
||||
curTime = self._core._utils.getCurrentHourEpoch
|
||||
curTime = self.getRoundedEpoch(rounding)
|
||||
self.nistSaltTimestamp = curTime
|
||||
data = self.doGetRequest('https://beacon.nist.gov/rest/record/' + str(curTime), port=torPort)
|
||||
dataXML = minidom.parseString(data, forbid_dtd=True, forbid_entities=True, forbid_external=True)
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@
|
|||
},
|
||||
|
||||
"allocations":{
|
||||
"disk": 1000000000,
|
||||
"disk": 9000000000,
|
||||
"netTotal": 1000000000,
|
||||
"blockCache" : 5000000,
|
||||
"blockCacheTotal" : 50000000
|
||||
|
|
|
|||
Loading…
Reference in New Issue