kev/Onionr
1
0
Fork 0
Code Issues 24 Pull Requests Projects Releases 2 Wiki Activity

Merge branch 'node-pow' into 'master'

Browse Source 
Reverse block insertion and other bug fixes

See merge request beardog/Onionr!4
This commit is contained in:
Kevin 2018-07-29 04:51:48 +00:00
commit 83c4dbcb72
9 changed files with 156 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
onionr/api.py
View File

@ -24,7 +24,7 @@ from gevent.wsgi import WSGIServer
import sys, random, threading, hmac, hashlib, base64, time, math, os, logger, config
from core import Core
from onionrblockapi import Block
import onionrutils, onionrcrypto
import onionrutils, onionrcrypto, blockimporter
class API:
'''
@ -141,9 +141,6 @@ class API:
resp = Response('Goodbye')
elif action == 'ping':
resp = Response('pong')
elif action == 'stats':
resp = Response('me_irl')
raise Exception
elif action == 'site':
block = data
siteData = self._core.getData(data)
@ -175,6 +172,24 @@ class API:
resp = Response("")
return resp
@app.route('/public/upload/', methods=['POST'])
def blockUpload():
self.validateHost('public')
resp = 'failure'
try:
data = request.form['block']
except KeyError:
logger.warn('No block specified for upload')
pass
else:
if sys.getsizeof(data) < 100000000:
if blockimporter.importBlockFromData(data, self._core):
resp = 'success'
else:
logger.warn('Error encountered importing uploaded block')
resp = Response(resp)
return resp
@app.route('/public/')
def public_handler():
# Public means it is publicly network accessible
@ -198,6 +213,7 @@ class API:
resp = Response('\n'.join(self._core.getBlockList()))
elif action == 'directMessage':
resp = Response(self._core.handle_direct_connection(data))
elif action == 'announce':
if data != '':
# TODO: require POW for this

40
onionr/blockimporter.py Normal file
View File

@ -0,0 +1,40 @@
'''
Onionr - P2P Microblogging Platform & Social network
Import block data and save it
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import core, onionrexceptions, logger
def importBlockFromData(content, coreInst):
retData = False
if not isinstance(coreInst, core.Core):
raise Exception("coreInst must be an Onionr core instance")
try:
content = content.encode()
except AttributeError:
pass
metas = coreInst._utils.getBlockMetadataFromData(content) # returns tuple(metadata, meta), meta is also in metadata
metadata = metas[0]
if coreInst._utils.validateMetadata(metadata): # check if metadata is valid
if coreInst._crypto.verifyPow(content): # check if POW is enough/correct
logger.info('Block passed proof, saving.')
blockHash = coreInst.setData(content)
blockHash = coreInst.addToBlockDB(blockHash, dataSaved=True)
coreInst._utils.processBlockMetadata(blockHash) # caches block metadata values to block database
retData = True
return retData

31
onionr/communicator2.py
View File

@ -36,6 +36,8 @@ class OnionrCommunicatorDaemon:
# intalize NIST beacon salt and time
self.nistSaltTimestamp = 0
self.powSalt = 0
self.blockToUpload = ''
# loop time.sleep delay in seconds
self.delay = 1
@ -84,7 +86,7 @@ class OnionrCommunicatorDaemon:
OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True)
# set loop to execute instantly to load up peer pool (replaced old pool init wait)
peerPoolTimer.count = (peerPoolTimer.frequency - 1)
peerPoolTimer.count = (peerPoolTimer.frequency - 1)
# Main daemon loop, mainly for calling timers, don't do any complex operations here to avoid locking
try:
@ -101,7 +103,7 @@ class OnionrCommunicatorDaemon:
logger.info('Goodbye.')
self._core._utils.localCommand('shutdown')
time.sleep(0.5)
def lookupKeys(self):
'''Lookup new keys'''
logger.debug('Looking up new keys...')
@ -111,7 +113,6 @@ class OnionrCommunicatorDaemon:
peer = self.pickOnlinePeer()
newKeys = self.peerAction(peer, action='kex')
self._core._utils.mergeKeys(newKeys)
self.decrementThreadCount('lookupKeys')
return
@ -196,7 +197,7 @@ class OnionrCommunicatorDaemon:
pass
logger.warn('Block hash validation failed for ' + blockHash + ' got ' + tempHash)
self.blockQueue.remove(blockHash) # remove from block queue both if success or false
self.currentDownloading.remove(blockHash)
self.currentDownloading.remove(blockHash)
self.decrementThreadCount('getBlocks')
return
@ -339,10 +340,32 @@ class OnionrCommunicatorDaemon:
for i in self.timers:
if i.timerFunction.__name__ == 'lookupKeys':
i.count = (i.frequency - 1)
elif cmd[0] == 'uploadBlock':
self.blockToUpload = cmd[1]
threading.Thread(target=self.uploadBlock).start()
else:
logger.info('Recieved daemonQueue command:' + cmd[0])
self.decrementThreadCount('daemonCommands')
def uploadBlock(self):
triedPeers = []
if not self._core._utils.validateHash(self.blockToUpload):
logger.warn('Requested to upload invalid block')
return
for i in range(max(len(self.onlinePeers), 2)):
peer = self.pickOnlinePeer()
if peer in triedPeers:
continue
triedPeers.append(peer)
url = 'http://' + peer + '/public/upload/'
data = {'block': block.Block(self.blockToUpload).getRaw()}
if peer.endswith('.onion'):
proxyType = 'tor'
elif peer.endswith('.i2p'):
proxyType = 'i2p'
logger.info("Uploading block")
self._core._utils.doPostRequest(url, data=data, proxyType=proxyType)
def announce(self, peer):
'''Announce to peers our address'''
announceCount = 0

24
onionr/core.py
View File

@ -41,10 +41,12 @@ class Core:
self.blockDataLocation = 'data/blocks/'
self.addressDB = 'data/address.db'
self.hsAdder = ''
self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt'
self.bootstrapList = []
self.requirements = onionrvalues.OnionrValues()
self.torPort = torPort
self.usageFile = 'data/disk-usage.txt'
if not os.path.exists('data/'):
os.mkdir('data/')
@ -578,25 +580,6 @@ class Core:
conn.close()
return
def handle_direct_connection(self, data):
'''
Handles direct messages
'''
try:
data = json.loads(data)
# TODO: Determine the sender, verify, etc
if ('callback' in data) and (data['callback'] is True):
# then this is a response to the message we sent earlier
self.daemonQueueAdd('checkCallbacks', json.dumps(data))
else:
# then we should handle it and respond accordingly
self.daemonQueueAdd('incomingDirectConnection', json.dumps(data))
except Exception as e:
logger.warn('Failed to handle incoming direct message: %s' % str(e))
return
def getBlockList(self, unsaved = False): # TODO: Use unsaved??
'''
Get list of our blocks
@ -757,6 +740,7 @@ class Core:
retData = self.setData(payload)
self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
self.setBlockType(retData, meta['type'])
self.daemonQueueAdd('uploadBlock', retData)
if retData != False:
events.event('insertBlock', onionr = None, threaded = False)

5
onionr/onionr.py
View File

@ -199,7 +199,7 @@ class Onionr:
'connect': self.addAddress,
'kex': self.doKEX,
'getpassword': self.getWebPassword
'getpassword': self.printWebPassword
}
self.cmdhelp = {
@ -258,6 +258,9 @@ class Onionr:
def getWebPassword(self):
return config.get('client.hmac')
def printWebPassword(self):
print(self.getWebPassword())
def getHelp(self):
return self.cmdhelp

4
onionr/onionrexceptions.py
View File

@ -42,6 +42,10 @@ class InvalidHexHash(Exception):
'''When a string is not a valid hex string of appropriate length for a hash value'''
pass
class InvalidProof(Exception):
'''When a proof is invalid or inadequate'''
pass
# network level exceptions
class MissingPort(Exception):
pass

13
onionr/onionrproofs.py
View File

@ -22,16 +22,19 @@ import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, lo
import core
class DataPOW:
def __init__(self, data, threadCount = 5):
def __init__(self, data, forceDifficulty=0, threadCount = 5):
self.foundHash = False
self.difficulty = 0
self.data = data
self.threadCount = threadCount
dataLen = sys.getsizeof(data)
self.difficulty = math.floor(dataLen / 1000000)
if self.difficulty <= 2:
self.difficulty = 4
if forceDifficulty == 0:
dataLen = sys.getsizeof(data)
self.difficulty = math.floor(dataLen / 1000000)
if self.difficulty <= 2:
self.difficulty = 4
else:
self.difficulty = forceDifficulty
try:
self.data = self.data.encode()

73
onionr/onionrutils.py
View File

@ -54,21 +54,12 @@ class OnionrUtils:
except Exception as error:
logger.error('Failed to fetch time bypass token.', error=error)
def sendPM(self, pubkey, message):
def getRoundedEpoch(self, roundS=60):
'''
High level function to encrypt a message to a peer and insert it as a block
'''
self._core.insertBlock(message, header='pm', sign=True, encryptType='asym', asymPeer=pubkey)
return
def getCurrentHourEpoch(self):
'''
Returns the current epoch, rounded down to the hour
Returns the epoch, rounded down to given seconds (Default 60)
'''
epoch = self.getEpoch()
return epoch - (epoch % 3600)
return epoch - (epoch % roundS)
def incrementAddressSuccess(self, address):
'''
@ -134,9 +125,10 @@ class OnionrUtils:
if newAdderList != False:
for adder in newAdderList.split(','):
if not adder in self._core.listAdders(randomOrder = False) and adder.strip() != self.getMyAddress():
if self._core.addAddress(adder):
logger.info('Added %s to db.' % adder, timestamp = True)
retVal = True
if adder[:4] == '0000':
if self._core.addAddress(adder):
logger.info('Added %s to db.' % adder, timestamp = True)
retVal = True
else:
pass
#logger.debug('%s is either our address or already in our DB' % adder)
@ -210,19 +202,26 @@ class OnionrUtils:
'''
meta = {}
metadata = {}
data = blockData
try:
blockData = blockData.encode()
except AttributeError:
pass
metadata = json.loads(blockData[:blockData.find(b'\n')].decode())
data = blockData[blockData.find(b'\n'):].decode()
try:
metadata = json.loads(blockData[:blockData.find(b'\n')].decode())
except json.decoder.JSONDecodeError:
pass
else:
data = blockData[blockData.find(b'\n'):].decode()
if not metadata['encryptType'] in ('asym', 'sym'):
try:
meta = json.loads(metadata['meta'])
except KeyError:
pass
meta = metadata['meta']
if not metadata['encryptType'] in ('asym', 'sym'):
try:
meta = json.loads(metadata['meta'])
except KeyError:
pass
meta = metadata['meta']
return (metadata, meta, data)
def checkPort(self, port, host=''):
@ -525,6 +524,30 @@ class OnionrUtils:
'''returns epoch'''
return math.floor(time.time())
def doPostRequest(self, url, data={}, port=0, proxyType='tor'):
'''
Do a POST request through a local tor or i2p instance
'''
if proxyType == 'tor':
if port == 0:
port = self._core.torPort
proxies = {'http': 'socks5://127.0.0.1:' + str(port), 'https': 'socks5://127.0.0.1:' + str(port)}
elif proxyType == 'i2p':
proxies = {'http': 'http://127.0.0.1:4444'}
else:
return
headers = {'user-agent': 'PyOnionr'}
try:
proxies = {'http': 'socks5h://127.0.0.1:' + str(port), 'https': 'socks5h://127.0.0.1:' + str(port)}
r = requests.post(url, data=data, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30))
retData = r.text
except KeyboardInterrupt:
raise KeyboardInterrupt
except requests.exceptions.RequestException as e:
logger.debug('Error: %s' % str(e))
retData = False
return retData
def doGetRequest(self, url, port=0, proxyType='tor'):
'''
Do a get request through a local tor or i2p instance
@ -549,7 +572,7 @@ class OnionrUtils:
retData = False
return retData
def getNistBeaconSalt(self, torPort=0):
def getNistBeaconSalt(self, torPort=0, rounding=3600):
'''
Get the token for the current hour from the NIST randomness beacon
'''
@ -559,7 +582,7 @@ class OnionrUtils:
except IndexError:
raise onionrexceptions.MissingPort('Missing Tor socks port')
retData = ''
curTime = self._core._utils.getCurrentHourEpoch
curTime = self.getRoundedEpoch(rounding)
self.nistSaltTimestamp = curTime
data = self.doGetRequest('https://beacon.nist.gov/rest/record/' + str(curTime), port=torPort)
dataXML = minidom.parseString(data, forbid_dtd=True, forbid_entities=True, forbid_external=True)

2
onionr/static-data/default_config.json
View File

@ -33,7 +33,7 @@
},
"allocations":{
"disk": 1000000000,
"disk": 9000000000,
"netTotal": 1000000000,
"blockCache" : 5000000,
"blockCacheTotal" : 50000000