diff --git a/onionr/api.py b/onionr/api.py index bf592c59..d6540b62 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -24,7 +24,7 @@ from gevent.wsgi import WSGIServer import sys, random, threading, hmac, hashlib, base64, time, math, os, logger, config from core import Core from onionrblockapi import Block -import onionrutils, onionrcrypto +import onionrutils, onionrcrypto, blockimporter class API: ''' @@ -141,9 +141,6 @@ class API: resp = Response('Goodbye') elif action == 'ping': resp = Response('pong') - elif action == 'stats': - resp = Response('me_irl') - raise Exception elif action == 'site': block = data siteData = self._core.getData(data) @@ -175,6 +172,24 @@ class API: resp = Response("") return resp + @app.route('/public/upload/', methods=['POST']) + def blockUpload(): + self.validateHost('public') + resp = 'failure' + try: + data = request.form['block'] + except KeyError: + logger.warn('No block specified for upload') + pass + else: + if sys.getsizeof(data) < 100000000: + if blockimporter.importBlockFromData(data, self._core): + resp = 'success' + else: + logger.warn('Error encountered importing uploaded block') + + resp = Response(resp) + return resp @app.route('/public/') def public_handler(): # Public means it is publicly network accessible @@ -198,6 +213,7 @@ class API: resp = Response('\n'.join(self._core.getBlockList())) elif action == 'directMessage': resp = Response(self._core.handle_direct_connection(data)) + elif action == 'announce': if data != '': # TODO: require POW for this diff --git a/onionr/blockimporter.py b/onionr/blockimporter.py new file mode 100644 index 00000000..a2695093 --- /dev/null +++ b/onionr/blockimporter.py @@ -0,0 +1,40 @@ +''' + Onionr - P2P Microblogging Platform & Social network + + Import block data and save it +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +import core, onionrexceptions, logger +def importBlockFromData(content, coreInst): + retData = False + if not isinstance(coreInst, core.Core): + raise Exception("coreInst must be an Onionr core instance") + + try: + content = content.encode() + except AttributeError: + pass + + metas = coreInst._utils.getBlockMetadataFromData(content) # returns tuple(metadata, meta), meta is also in metadata + metadata = metas[0] + if coreInst._utils.validateMetadata(metadata): # check if metadata is valid + if coreInst._crypto.verifyPow(content): # check if POW is enough/correct + logger.info('Block passed proof, saving.') + blockHash = coreInst.setData(content) + blockHash = coreInst.addToBlockDB(blockHash, dataSaved=True) + coreInst._utils.processBlockMetadata(blockHash) # caches block metadata values to block database + retData = True + return retData \ No newline at end of file diff --git a/onionr/communicator2.py b/onionr/communicator2.py index b4a32649..ba24991b 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -36,6 +36,8 @@ class OnionrCommunicatorDaemon: # intalize NIST beacon salt and time self.nistSaltTimestamp = 0 self.powSalt = 0 + + self.blockToUpload = '' # loop time.sleep delay in seconds self.delay = 1 @@ -84,7 +86,7 @@ class OnionrCommunicatorDaemon: OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True) # set loop to execute instantly to load up peer pool (replaced old pool init wait) - peerPoolTimer.count = (peerPoolTimer.frequency - 1) + peerPoolTimer.count = (peerPoolTimer.frequency - 1) # Main daemon loop, mainly for calling timers, don't do any complex operations here to avoid locking try: @@ -101,7 +103,7 @@ class OnionrCommunicatorDaemon: logger.info('Goodbye.') self._core._utils.localCommand('shutdown') time.sleep(0.5) - + def lookupKeys(self): '''Lookup new keys''' logger.debug('Looking up new keys...') @@ -111,7 +113,6 @@ class OnionrCommunicatorDaemon: peer = self.pickOnlinePeer() newKeys = self.peerAction(peer, action='kex') self._core._utils.mergeKeys(newKeys) - self.decrementThreadCount('lookupKeys') return @@ -196,7 +197,7 @@ class OnionrCommunicatorDaemon: pass logger.warn('Block hash validation failed for ' + blockHash + ' got ' + tempHash) self.blockQueue.remove(blockHash) # remove from block queue both if success or false - self.currentDownloading.remove(blockHash) + self.currentDownloading.remove(blockHash) self.decrementThreadCount('getBlocks') return @@ -339,10 +340,32 @@ class OnionrCommunicatorDaemon: for i in self.timers: if i.timerFunction.__name__ == 'lookupKeys': i.count = (i.frequency - 1) + elif cmd[0] == 'uploadBlock': + self.blockToUpload = cmd[1] + threading.Thread(target=self.uploadBlock).start() else: logger.info('Recieved daemonQueue command:' + cmd[0]) self.decrementThreadCount('daemonCommands') + def uploadBlock(self): + triedPeers = [] + if not self._core._utils.validateHash(self.blockToUpload): + logger.warn('Requested to upload invalid block') + return + for i in range(max(len(self.onlinePeers), 2)): + peer = self.pickOnlinePeer() + if peer in triedPeers: + continue + triedPeers.append(peer) + url = 'http://' + peer + '/public/upload/' + data = {'block': block.Block(self.blockToUpload).getRaw()} + if peer.endswith('.onion'): + proxyType = 'tor' + elif peer.endswith('.i2p'): + proxyType = 'i2p' + logger.info("Uploading block") + self._core._utils.doPostRequest(url, data=data, proxyType=proxyType) + def announce(self, peer): '''Announce to peers our address''' announceCount = 0 diff --git a/onionr/core.py b/onionr/core.py index fbe9cc80..b8b33c73 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -41,10 +41,12 @@ class Core: self.blockDataLocation = 'data/blocks/' self.addressDB = 'data/address.db' self.hsAdder = '' - self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt' self.bootstrapList = [] self.requirements = onionrvalues.OnionrValues() + self.torPort = torPort + + self.usageFile = 'data/disk-usage.txt' if not os.path.exists('data/'): os.mkdir('data/') @@ -578,25 +580,6 @@ class Core: conn.close() return - def handle_direct_connection(self, data): - ''' - Handles direct messages - ''' - try: - data = json.loads(data) - - # TODO: Determine the sender, verify, etc - if ('callback' in data) and (data['callback'] is True): - # then this is a response to the message we sent earlier - self.daemonQueueAdd('checkCallbacks', json.dumps(data)) - else: - # then we should handle it and respond accordingly - self.daemonQueueAdd('incomingDirectConnection', json.dumps(data)) - except Exception as e: - logger.warn('Failed to handle incoming direct message: %s' % str(e)) - - return - def getBlockList(self, unsaved = False): # TODO: Use unsaved?? ''' Get list of our blocks @@ -757,6 +740,7 @@ class Core: retData = self.setData(payload) self.addToBlockDB(retData, selfInsert=True, dataSaved=True) self.setBlockType(retData, meta['type']) + self.daemonQueueAdd('uploadBlock', retData) if retData != False: events.event('insertBlock', onionr = None, threaded = False) diff --git a/onionr/onionr.py b/onionr/onionr.py index 10aa54ce..11db4351 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -199,7 +199,7 @@ class Onionr: 'connect': self.addAddress, 'kex': self.doKEX, - 'getpassword': self.getWebPassword + 'getpassword': self.printWebPassword } self.cmdhelp = { @@ -258,6 +258,9 @@ class Onionr: def getWebPassword(self): return config.get('client.hmac') + + def printWebPassword(self): + print(self.getWebPassword()) def getHelp(self): return self.cmdhelp diff --git a/onionr/onionrexceptions.py b/onionr/onionrexceptions.py index c97849c1..d0a6d248 100644 --- a/onionr/onionrexceptions.py +++ b/onionr/onionrexceptions.py @@ -42,6 +42,10 @@ class InvalidHexHash(Exception): '''When a string is not a valid hex string of appropriate length for a hash value''' pass +class InvalidProof(Exception): + '''When a proof is invalid or inadequate''' + pass + # network level exceptions class MissingPort(Exception): pass diff --git a/onionr/onionrproofs.py b/onionr/onionrproofs.py index 194b37e9..b93d5724 100644 --- a/onionr/onionrproofs.py +++ b/onionr/onionrproofs.py @@ -22,16 +22,19 @@ import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, lo import core class DataPOW: - def __init__(self, data, threadCount = 5): + def __init__(self, data, forceDifficulty=0, threadCount = 5): self.foundHash = False self.difficulty = 0 self.data = data self.threadCount = threadCount - dataLen = sys.getsizeof(data) - self.difficulty = math.floor(dataLen / 1000000) - if self.difficulty <= 2: - self.difficulty = 4 + if forceDifficulty == 0: + dataLen = sys.getsizeof(data) + self.difficulty = math.floor(dataLen / 1000000) + if self.difficulty <= 2: + self.difficulty = 4 + else: + self.difficulty = forceDifficulty try: self.data = self.data.encode() diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 33e3f51d..2b6c6e5a 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -54,21 +54,12 @@ class OnionrUtils: except Exception as error: logger.error('Failed to fetch time bypass token.', error=error) - def sendPM(self, pubkey, message): + def getRoundedEpoch(self, roundS=60): ''' - High level function to encrypt a message to a peer and insert it as a block - ''' - - self._core.insertBlock(message, header='pm', sign=True, encryptType='asym', asymPeer=pubkey) - - return - - def getCurrentHourEpoch(self): - ''' - Returns the current epoch, rounded down to the hour + Returns the epoch, rounded down to given seconds (Default 60) ''' epoch = self.getEpoch() - return epoch - (epoch % 3600) + return epoch - (epoch % roundS) def incrementAddressSuccess(self, address): ''' @@ -134,9 +125,10 @@ class OnionrUtils: if newAdderList != False: for adder in newAdderList.split(','): if not adder in self._core.listAdders(randomOrder = False) and adder.strip() != self.getMyAddress(): - if self._core.addAddress(adder): - logger.info('Added %s to db.' % adder, timestamp = True) - retVal = True + if adder[:4] == '0000': + if self._core.addAddress(adder): + logger.info('Added %s to db.' % adder, timestamp = True) + retVal = True else: pass #logger.debug('%s is either our address or already in our DB' % adder) @@ -210,19 +202,26 @@ class OnionrUtils: ''' meta = {} + metadata = {} + data = blockData try: blockData = blockData.encode() except AttributeError: pass - metadata = json.loads(blockData[:blockData.find(b'\n')].decode()) - data = blockData[blockData.find(b'\n'):].decode() + + try: + metadata = json.loads(blockData[:blockData.find(b'\n')].decode()) + except json.decoder.JSONDecodeError: + pass + else: + data = blockData[blockData.find(b'\n'):].decode() - if not metadata['encryptType'] in ('asym', 'sym'): - try: - meta = json.loads(metadata['meta']) - except KeyError: - pass - meta = metadata['meta'] + if not metadata['encryptType'] in ('asym', 'sym'): + try: + meta = json.loads(metadata['meta']) + except KeyError: + pass + meta = metadata['meta'] return (metadata, meta, data) def checkPort(self, port, host=''): @@ -525,6 +524,30 @@ class OnionrUtils: '''returns epoch''' return math.floor(time.time()) + def doPostRequest(self, url, data={}, port=0, proxyType='tor'): + ''' + Do a POST request through a local tor or i2p instance + ''' + if proxyType == 'tor': + if port == 0: + port = self._core.torPort + proxies = {'http': 'socks5://127.0.0.1:' + str(port), 'https': 'socks5://127.0.0.1:' + str(port)} + elif proxyType == 'i2p': + proxies = {'http': 'http://127.0.0.1:4444'} + else: + return + headers = {'user-agent': 'PyOnionr'} + try: + proxies = {'http': 'socks5h://127.0.0.1:' + str(port), 'https': 'socks5h://127.0.0.1:' + str(port)} + r = requests.post(url, data=data, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30)) + retData = r.text + except KeyboardInterrupt: + raise KeyboardInterrupt + except requests.exceptions.RequestException as e: + logger.debug('Error: %s' % str(e)) + retData = False + return retData + def doGetRequest(self, url, port=0, proxyType='tor'): ''' Do a get request through a local tor or i2p instance @@ -549,7 +572,7 @@ class OnionrUtils: retData = False return retData - def getNistBeaconSalt(self, torPort=0): + def getNistBeaconSalt(self, torPort=0, rounding=3600): ''' Get the token for the current hour from the NIST randomness beacon ''' @@ -559,7 +582,7 @@ class OnionrUtils: except IndexError: raise onionrexceptions.MissingPort('Missing Tor socks port') retData = '' - curTime = self._core._utils.getCurrentHourEpoch + curTime = self.getRoundedEpoch(rounding) self.nistSaltTimestamp = curTime data = self.doGetRequest('https://beacon.nist.gov/rest/record/' + str(curTime), port=torPort) dataXML = minidom.parseString(data, forbid_dtd=True, forbid_entities=True, forbid_external=True) diff --git a/onionr/static-data/default_config.json b/onionr/static-data/default_config.json index 9188aa66..db86bbe5 100644 --- a/onionr/static-data/default_config.json +++ b/onionr/static-data/default_config.json @@ -33,7 +33,7 @@ }, "allocations":{ - "disk": 1000000000, + "disk": 9000000000, "netTotal": 1000000000, "blockCache" : 5000000, "blockCacheTotal" : 50000000