diff --git a/onionr/api.py b/onionr/api.py index 3af4add9..6e637136 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -94,6 +94,8 @@ class API: self.mimeType = 'text/plain' self.overrideCSP = False + self.hideBlocks = [] # Blocks to be denied sharing + with open(self._core.dataDir + 'time-bypass.txt', 'w') as bypass: bypass.write(self.timeBypassToken) @@ -231,6 +233,15 @@ class API: self.validateHost('private') if action == 'hello': resp = Response('Hello, World! ' + request.host) + elif action == 'waitForShare': + if self._core._utils.validateHash(data): + if data not in self.hideBlocks: + self.hideBlocks.append(data) + else: + self.hideBlocks.remove(data) + resp = "success" + else: + resp = "failed to validate hash" elif action == 'shutdown': # request.environ.get('werkzeug.server.shutdown')() self.http_server.stop() @@ -469,7 +480,11 @@ class API: elif action == 'getDBHash': resp = Response(self._utils.getBlockDBHash()) elif action == 'getBlockHashes': - resp = Response('\n'.join(self._core.getBlockList())) + bList = self._core.getBlockList() + for b in self.hideBlocks: + if b in bList: + bList.remove(b) + resp = Response('\n'.join(bList)) # setData should be something the communicator initiates, not this api elif action == 'getData': resp = '' diff --git a/onionr/communicator2.py b/onionr/communicator2.py index d57b7b66..08527f8f 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -43,7 +43,7 @@ class OnionrCommunicatorDaemon: self.nistSaltTimestamp = 0 self.powSalt = 0 - self.blockToUpload = '' + self.blocksToUpload = [] # loop time.sleep delay in seconds self.delay = 1 @@ -89,16 +89,15 @@ class OnionrCommunicatorDaemon: # Set timers, function reference, seconds # requiresPeer True means the timer function won't fire if we have no connected peers - OnionrCommunicatorTimers(self, self.runCheck, 1) - OnionrCommunicatorTimers(self, self.daemonCommands, 5) - OnionrCommunicatorTimers(self, self.detectAPICrash, 5) peerPoolTimer = OnionrCommunicatorTimers(self, self.getOnlinePeers, 60, maxThreads=1) - OnionrCommunicatorTimers(self, self.lookupBlocks, self._core.config.get('timers.lookup_blocks'), requiresPeer=True, maxThreads=1) - OnionrCommunicatorTimers(self, self.getBlocks, self._core.config.get('timers.get_blocks'), requiresPeer=True) + OnionrCommunicatorTimers(self, self.runCheck, 1) + OnionrCommunicatorTimers(self, self.lookupBlocks, self._core.config.get('timers.lookupBlocks'), requiresPeer=True, maxThreads=1) + OnionrCommunicatorTimers(self, self.getBlocks, self._core.config.get('timers.getBlocks'), requiresPeer=True) OnionrCommunicatorTimers(self, self.clearOfflinePeer, 58) OnionrCommunicatorTimers(self, self.daemonTools.cleanOldBlocks, 65) OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True) OnionrCommunicatorTimers(self, self.daemonTools.cooldownPeer, 30, requiresPeer=True) + OnionrCommunicatorTimers(self, self.uploadBlock, 10, requiresPeer=True, maxThreads=1) netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600) announceTimer = OnionrCommunicatorTimers(self, self.daemonTools.announceNode, 305, requiresPeer=True, maxThreads=1) cleanupTimer = OnionrCommunicatorTimers(self, self.peerCleanup, 300, requiresPeer=True) @@ -170,7 +169,7 @@ class OnionrCommunicatorDaemon: else: continue newDBHash = self.peerAction(peer, 'getDBHash') # get their db hash - if newDBHash == False: + if newDBHash == False or not self._core._utils.validateHash(newDBHash): continue # if request failed, restart loop (peer is added to offline peers automatically) triedPeers.append(peer) if newDBHash != self._core.getAddressInfo(peer, 'DBHash'): @@ -409,7 +408,7 @@ class OnionrCommunicatorDaemon: '''Perform a get request to a peer''' if len(peer) == 0: return False - logger.info('Performing ' + action + ' with ' + peer + ' on port ' + str(self.proxyPort)) + #logger.debug('Performing ' + action + ' with ' + peer + ' on port ' + str(self.proxyPort)) url = 'http://' + peer + '/public/?action=' + action if len(data) > 0: url += '&data=' + data @@ -475,8 +474,7 @@ class OnionrCommunicatorDaemon: if i.timerFunction.__name__ == 'lookupAdders': i.count = (i.frequency - 1) elif cmd[0] == 'uploadBlock': - self.blockToUpload = cmd[1] - threading.Thread(target=self.uploadBlock).start() + self.blocksToUpload.append(cmd[1]) elif cmd[0] == 'startSocket': # Create our own socket server socketInfo = json.loads(cmd[1]) @@ -497,23 +495,36 @@ class OnionrCommunicatorDaemon: '''Upload our block to a few peers''' # when inserting a block, we try to upload it to a few peers to add some deniability triedPeers = [] - if not self._core._utils.validateHash(self.blockToUpload): - logger.warn('Requested to upload invalid block') - return - for i in range(max(len(self.onlinePeers), 2)): - peer = self.pickOnlinePeer() - if peer in triedPeers: - continue - triedPeers.append(peer) - url = 'http://' + peer + '/public/upload/' - data = {'block': block.Block(self.blockToUpload).getRaw()} - proxyType = '' - if peer.endswith('.onion'): - proxyType = 'tor' - elif peer.endswith('.i2p'): - proxyType = 'i2p' - logger.info("Uploading block") - self._core._utils.doPostRequest(url, data=data, proxyType=proxyType) + finishedUploads = [] + if len(self.blocksToUpload) != 0: + for bl in self.blocksToUpload: + if not self._core._utils.validateHash(bl): + logger.warn('Requested to upload invalid block') + self.decrementThreadCount('uploadBlock') + return + for i in range(max(len(self.onlinePeers), 2)): + peer = self.pickOnlinePeer() + if peer in triedPeers: + continue + triedPeers.append(peer) + url = 'http://' + peer + '/public/upload/' + data = {'block': block.Block(bl).getRaw()} + proxyType = '' + if peer.endswith('.onion'): + proxyType = 'tor' + elif peer.endswith('.i2p'): + proxyType = 'i2p' + logger.info("Uploading block to " + peer) + if not self._core._utils.doPostRequest(url, data=data, proxyType=proxyType) == False: + self._core._utils.localCommand('waitForShare', data=bl) + finishedUploads.append(bl) + break + for x in finishedUploads: + try: + self.blocksToUpload.remove(x) + except ValueError: + pass + self.decrementThreadCount('uploadBlock') def announce(self, peer): '''Announce to peers our address''' diff --git a/onionr/core.py b/onionr/core.py index 5e069a22..781c6527 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -592,7 +592,6 @@ class Core: ''' conn = sqlite3.connect(self.blockDB, timeout=10) - c = conn.cursor() execute = 'SELECT dateReceived FROM hashes WHERE hash=?;' @@ -609,7 +608,6 @@ class Core: ''' conn = sqlite3.connect(self.blockDB, timeout=10) - c = conn.cursor() if orderDate: @@ -623,6 +621,7 @@ class Core: for row in c.execute(execute, args): for i in row: rows.append(i) + return rows def getExpiredBlocks(self): @@ -631,10 +630,10 @@ class Core: c = conn.cursor() date = int(self._utils.getEpoch()) - execute = 'SELECT hash FROM hashes WHERE expire <= ? ORDER BY dateReceived;' + execute = 'SELECT hash FROM hashes WHERE expire <= %s ORDER BY dateReceived;' % (date,) rows = list() - for row in c.execute(execute, (date,)): + for row in c.execute(execute): for i in row: rows.append(i) return rows @@ -680,8 +679,7 @@ class Core: return True - - def insertBlock(self, data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = dict(), expire=None): + def insertBlock(self, data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = None, expire=None): ''' Inserts a block into the network encryptType must be specified to encrypt a block @@ -784,6 +782,8 @@ class Core: payload = proof.waitForResult() if payload != False: retData = self.setData(payload) + # Tell the api server through localCommand to wait for the daemon to upload this block to make stastical analysis more difficult + self._utils.localCommand('waitForShare', data=retData) self.addToBlockDB(retData, selfInsert=True, dataSaved=True) #self.setBlockType(retData, meta['type']) self._utils.processBlockMetadata(retData) diff --git a/onionr/onionr.py b/onionr/onionr.py index 819a3d8d..d7117421 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -40,7 +40,7 @@ except ImportError: raise Exception("You need the PySocks module (for use with socks5 proxy to use Tor)") ONIONR_TAGLINE = 'Anonymous P2P Platform - GPLv3 - https://Onionr.VoidNet.Tech' -ONIONR_VERSION = '0.3.0' # for debugging and stuff +ONIONR_VERSION = '0.3.2' # for debugging and stuff ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION) API_VERSION = '5' # increments of 1; only change when something fundemental about how the API works changes. This way other nodes know how to communicate without learning too much information about you. @@ -64,7 +64,6 @@ class Onionr: self.dataDir = 'data/' # Load global configuration data - data_exists = Onionr.setupConfig(self.dataDir, self = self) self.onionrCore = core.Core() @@ -169,6 +168,7 @@ class Onionr: 'add-file': self.addFile, 'addfile': self.addFile, + 'get-file': self.getFile, 'getfile': self.getFile, @@ -186,6 +186,17 @@ class Onionr: 'gui' : self.openUI, 'chat': self.startChat, + 'getpassword': self.printWebPassword, + 'get-password': self.printWebPassword, + 'getpwd': self.printWebPassword, + 'get-pwd': self.printWebPassword, + 'getpass': self.printWebPassword, + 'get-pass': self.printWebPassword, + 'getpasswd': self.printWebPassword, + 'get-passwd': self.printWebPassword, + + 'chat': self.startChat, + 'friend': self.friendCmd } @@ -246,7 +257,6 @@ class Onionr: for detail in details: logger.info('%s%s: \n%s%s\n' % (logger.colors.fg.lightgreen, detail, logger.colors.fg.green, details[detail]), sensitive = True) - def startChat(self): try: data = json.dumps({'peer': sys.argv[2], 'reason': 'chat'}) @@ -300,6 +310,48 @@ class Onionr: logger.info('Syntax: friend add/remove/list [address]') + def friendCmd(self): + '''List, add, or remove friend(s) + Changes their peer DB entry. + ''' + friend = '' + try: + # Get the friend command + action = sys.argv[2] + except IndexError: + logger.info('Syntax: friend add/remove/list [address]') + else: + action = action.lower() + if action == 'list': + # List out peers marked as our friend + for friend in self.onionrCore.listPeers(randomOrder=False, trust=1): + if friend == self.onionrCore._crypto.pubKey: # do not list our key + continue + friendProfile = onionrusers.OnionrUser(self.onionrCore, friend) + logger.info(friend + ' - ' + friendProfile.getName()) + elif action in ('add', 'remove'): + try: + friend = sys.argv[3] + if not self.onionrUtils.validatePubKey(friend): + raise onionrexceptions.InvalidPubkey('Public key is invalid') + if friend not in self.onionrCore.listPeers(): + raise onionrexceptions.KeyNotKnown + friend = onionrusers.OnionrUser(self.onionrCore, friend) + except IndexError: + logger.error('Friend ID is required.') + except onionrexceptions.KeyNotKnown: + logger.error('That peer is not in our database') + else: + if action == 'add': + friend.setTrust(1) + logger.info('Added %s as friend.' % (friend.publicKey,)) + else: + friend.setTrust(0) + logger.info('Removed %s as friend.' % (friend.publicKey,)) + else: + logger.info('Syntax: friend add/remove/list [address]') + + def banBlock(self): try: ban = sys.argv[2] @@ -633,6 +685,7 @@ class Onionr: logger.debug('Started .onion service: %s' % (logger.colors.underline + net.myID)) logger.debug('Using public key: %s' % (logger.colors.underline + self.onionrCore._crypto.pubKey)) time.sleep(1) + # TODO: make runable on windows communicatorProc = subprocess.Popen([communicatorDaemon, 'run', str(net.socksPort)]) @@ -798,6 +851,7 @@ class Onionr: logger.error("Syntax %s %s" % (sys.argv[0], '/path/to/filename ')) else: logger.info(fileName) + contents = None if os.path.exists(fileName): logger.error("File already exists") @@ -805,6 +859,7 @@ class Onionr: if not self.onionrUtils.validateHash(bHash): logger.error('Block hash is invalid') return + Block.mergeChain(bHash, fileName) return diff --git a/onionr/onionrblacklist.py b/onionr/onionrblacklist.py index f5b9925c..8736f78f 100644 --- a/onionr/onionrblacklist.py +++ b/onionr/onionrblacklist.py @@ -30,13 +30,16 @@ class OnionrBlackList: def inBlacklist(self, data): hashed = self._core._utils.bytesToStr(self._core._crypto.sha3Hash(data)) retData = False + if not hashed.isalnum(): raise Exception("Hashed data is not alpha numeric") if len(hashed) > 64: raise Exception("Hashed data is too large") + for i in self._dbExecute("SELECT * FROM blacklist WHERE hash = ?", (hashed,)): retData = True # this only executes if an entry is present by that hash break + return retData def _dbExecute(self, toExec, params = ()): diff --git a/onionr/onionrblockapi.py b/onionr/onionrblockapi.py index eba350b7..e6506faf 100644 --- a/onionr/onionrblockapi.py +++ b/onionr/onionrblockapi.py @@ -243,10 +243,10 @@ class Block: if (not self.getBlockFile() is None) and (recreate is True): with open(self.getBlockFile(), 'wb') as blockFile: blockFile.write(self.getRaw().encode()) - self.update() else: self.hash = self.getCore().insertBlock(self.getContent(), header = self.getType(), sign = sign, meta = self.getMetadata(), expire = self.getExpire()) - self.update() + + self.update() return self.getHash() else: diff --git a/onionr/onionrchat.py b/onionr/onionrchat.py index 782266f8..84483295 100644 --- a/onionr/onionrchat.py +++ b/onionr/onionrchat.py @@ -46,4 +46,5 @@ class OnionrChat: self.communicator.socketClient.sendData(peer, "lol") except: pass + time.sleep(2) diff --git a/onionr/onionrdaemontools.py b/onionr/onionrdaemontools.py index 3adeb5ea..28d7f964 100644 --- a/onionr/onionrdaemontools.py +++ b/onionr/onionrdaemontools.py @@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' + import onionrexceptions, onionrpeers, onionrproofs, logger, onionrusers import base64, sqlite3, os from dependencies import secrets @@ -89,6 +90,7 @@ class DaemonTools: c = conn.cursor() time = self.daemon._core._utils.getEpoch() deleteKeys = [] + for entry in c.execute("SELECT * FROM forwardKeys WHERE expire <= ?", (time,)): logger.info(entry[1]) deleteKeys.append(entry[1]) @@ -120,6 +122,7 @@ class DaemonTools: # Cool down a peer, if we have max connections alive for long enough if onlinePeerAmount >= self.daemon._core.config.get('peers.max_connect', 10): finding = True + while finding: try: toCool = min(tempConnectTimes, key=tempConnectTimes.get) @@ -132,6 +135,7 @@ class DaemonTools: else: self.daemon.removeOnlinePeer(toCool) self.daemon.cooldownPeer[toCool] = self.daemon._core._utils.getEpoch() + self.daemon.decrementThreadCount('cooldownPeer') def runCheck(self): diff --git a/onionr/onionrsockets.py b/onionr/onionrsockets.py index 6ff08366..aa02c3db 100644 --- a/onionr/onionrsockets.py +++ b/onionr/onionrsockets.py @@ -87,6 +87,7 @@ class OnionrSocketServer: def detectShutdown(self): while not self._core.killSockets: time.sleep(5) + logger.debug('Killing socket server...') self.http_server.stop() diff --git a/onionr/onionrusers.py b/onionr/onionrusers.py index 3dd538af..b9bc3c2c 100644 --- a/onionr/onionrusers.py +++ b/onionr/onionrusers.py @@ -107,12 +107,14 @@ class OnionrUser: conn.commit() conn.close() + return key def _getForwardKeys(self): conn = sqlite3.connect(self._core.peerDB, timeout=10) c = conn.cursor() keyList = [] + for row in c.execute("SELECT forwardKey FROM forwardKeys WHERE peerKey = ? ORDER BY date DESC", (self.publicKey,)): key = row[0] keyList.append(key) @@ -150,8 +152,10 @@ class OnionrUser: pubkey = self._core._utils.bytesToStr(pubkey) command = (pubkey,) keyList = [] # list of tuples containing pub, private for peer + for result in c.execute("SELECT * FROM myForwardKeys WHERE peer = ?", command): keyList.append((result[1], result[2])) + if len(keyList) == 0: if genNew: self.generateForwardKey() diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 956b2e22..b185b0bb 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -18,7 +18,7 @@ along with this program. If not, see . ''' # Misc functions that do not fit in the main api, but are useful -import getpass, sys, requests, os, socket, hashlib, logger, sqlite3, config, binascii, time, base64, json, glob, shutil, math, json, re +import getpass, sys, requests, os, socket, hashlib, logger, sqlite3, config, binascii, time, base64, json, glob, shutil, math, json, re, urllib.parse import nacl.signing, nacl.encoding from onionrblockapi import Block import onionrexceptions @@ -150,7 +150,7 @@ class OnionrUtils: logger.error('Failed to read my address.', error = error) return None - def localCommand(self, command, silent = True): + def localCommand(self, command, data='', silent = True): ''' Send a command to the local http API server, securely. Intended for local clients, DO NOT USE for remote peers. ''' @@ -164,6 +164,8 @@ class OnionrUtils: except FileNotFoundError: return False payload = 'http://%s:%s/client/?action=%s&token=%s&timingToken=%s' % (hostname, config.get('client.port'), command, config.get('client.hmac'), self.timingToken) + if data != '': + payload += '&data=' + urllib.parse.quote_plus(data) try: retData = requests.get(payload).text except Exception as error: @@ -495,6 +497,12 @@ class OnionrUtils: except binascii.Error: retVal = False + # Validate address is valid base32 (when capitalized and minus extension); v2/v3 onions and .b32.i2p use base32 + try: + base64.b32decode(idNoDomain.upper().encode()) + except binascii.Error: + retVal = False + return retVal except: return False diff --git a/onionr/static-data/default-plugins/cliui/main.py b/onionr/static-data/default-plugins/cliui/main.py index c29465f4..f9fb691f 100644 --- a/onionr/static-data/default-plugins/cliui/main.py +++ b/onionr/static-data/default-plugins/cliui/main.py @@ -52,7 +52,7 @@ class OnionrCLIUI: while showMenu: if firstRun: - logger.info('Please wait while Onionr starts...'') + logger.info('Please wait while Onionr starts...') daemon = subprocess.Popen(["./onionr.py", "start"], stdin=subprocess.PIPE, stdout=subprocess.DEVNULL) time.sleep(30) firstRun = False @@ -63,7 +63,6 @@ class OnionrCLIUI: isOnline = "No" logger.info('''Daemon Running: ''' + isOnline + ''' - 1. Flow (Anonymous public chat, use at your own risk) 2. Mail (Secure email-like service) 3. File Sharing @@ -91,6 +90,7 @@ class OnionrCLIUI: if isOnline == "Yes": logger.info("Onionr daemon will shutdown...") self.myCore.daemonQueueAdd('shutdown') + try: daemon.kill() except UnboundLocalError: diff --git a/onionr/static-data/default-plugins/encrypt/main.py b/onionr/static-data/default-plugins/encrypt/main.py index 299f202e..01eabd81 100644 --- a/onionr/static-data/default-plugins/encrypt/main.py +++ b/onionr/static-data/default-plugins/encrypt/main.py @@ -72,6 +72,7 @@ class PlainEncryption: encrypted = self.api.get_core()._crypto.pubKeyEncrypt(plaintext, pubkey, anonymous=True, encodedData=True) encrypted = self.api.get_core()._utils.bytesToStr(encrypted) logger.info('Encrypted Message: \n\nONIONR ENCRYPTED DATA %s END ENCRYPTED DATA' % (encrypted,)) + def decrypt(self): plaintext = "" data = "" @@ -102,7 +103,6 @@ class PlainEncryption: logger.info("Message has good signature.") return - def on_init(api, data = None): ''' This event is called after Onionr is initialized, but before the command @@ -114,4 +114,5 @@ def on_init(api, data = None): encrypt = PlainEncryption(pluginapi) api.commands.register(['encrypt'], encrypt.encrypt) api.commands.register(['decrypt'], encrypt.decrypt) + return diff --git a/onionr/static-data/default-plugins/pms/main.py b/onionr/static-data/default-plugins/pms/main.py index 45abd506..5ba910eb 100644 --- a/onionr/static-data/default-plugins/pms/main.py +++ b/onionr/static-data/default-plugins/pms/main.py @@ -129,8 +129,10 @@ class OnionrMail: else: cancel = '' readBlock.verifySig() + logger.info('Message recieved from %s' % (self.myCore._utils.bytesToStr(readBlock.signer,))) logger.info('Valid signature: %s' % readBlock.validSig) + if not readBlock.validSig: logger.warn('This message has an INVALID signature. ANYONE could have sent this message.') cancel = logger.readline('Press enter to continue to message, or -q to not open the message (recommended).') @@ -172,6 +174,7 @@ class OnionrMail: for i in self.sentboxTools.listSent(): self.sentboxList.append(i['hash']) self.sentMessages[i['hash']] = (i['message'], i['peer']) + logger.info('%s. %s - %s - %s' % (count, i['hash'], i['peer'][:12], i['date'])) count += 1 diff --git a/onionr/static-data/default_config.json b/onionr/static-data/default_config.json index d2cde4c8..82dbece0 100644 --- a/onionr/static-data/default_config.json +++ b/onionr/static-data/default_config.json @@ -2,6 +2,8 @@ "general" : { "dev_mode" : true, "display_header" : true, + "minimum_block_pow": 5, + "minimum_send_pow": 5, "minimum_block_pow": 5, "minimum_send_pow": 5,