Fix merge conflicts

onionr/api.py

@@ -94,6 +94,8 @@ class API:
         self.mimeType = 'text/plain'
         self.overrideCSP = False
 
+        self.hideBlocks = [] # Blocks to be denied sharing
+
         with open(self._core.dataDir + 'time-bypass.txt', 'w') as bypass:
             bypass.write(self.timeBypassToken)
 
@@ -231,6 +233,15 @@ class API:
             self.validateHost('private')
             if action == 'hello':
                 resp = Response('Hello, World! ' + request.host)
+            elif action == 'waitForShare':
+                if self._core._utils.validateHash(data):
+                    if data not in self.hideBlocks:
+                        self.hideBlocks.append(data)
+                    else:
+                        self.hideBlocks.remove(data)
+                    resp = "success"
+                else:
+                    resp = "failed to validate hash"
             elif action == 'shutdown':
                 # request.environ.get('werkzeug.server.shutdown')()
                 self.http_server.stop()
@@ -469,7 +480,11 @@ class API:
             elif action == 'getDBHash':
                 resp = Response(self._utils.getBlockDBHash())
             elif action == 'getBlockHashes':
-                resp = Response('\n'.join(self._core.getBlockList()))
+                bList = self._core.getBlockList()
+                for b in self.hideBlocks:
+                    if b in bList:
+                        bList.remove(b)
+                resp = Response('\n'.join(bList))
             # setData should be something the communicator initiates, not this api
             elif action == 'getData':
                 resp = ''

onionr/communicator2.py

@@ -43,7 +43,7 @@ class OnionrCommunicatorDaemon:
         self.nistSaltTimestamp = 0
         self.powSalt = 0
 
-        self.blockToUpload = ''
+        self.blocksToUpload = []
 
         # loop time.sleep delay in seconds
         self.delay = 1
@@ -89,16 +89,15 @@ class OnionrCommunicatorDaemon:
 
         # Set timers, function reference, seconds
         # requiresPeer True means the timer function won't fire if we have no connected peers
-        OnionrCommunicatorTimers(self, self.runCheck, 1)
-        OnionrCommunicatorTimers(self, self.daemonCommands, 5)
-        OnionrCommunicatorTimers(self, self.detectAPICrash, 5)
         peerPoolTimer = OnionrCommunicatorTimers(self, self.getOnlinePeers, 60, maxThreads=1)
-        OnionrCommunicatorTimers(self, self.lookupBlocks, self._core.config.get('timers.lookup_blocks'), requiresPeer=True, maxThreads=1)
+        OnionrCommunicatorTimers(self, self.runCheck, 1)
-        OnionrCommunicatorTimers(self, self.getBlocks, self._core.config.get('timers.get_blocks'), requiresPeer=True)
+        OnionrCommunicatorTimers(self, self.lookupBlocks, self._core.config.get('timers.lookupBlocks'), requiresPeer=True, maxThreads=1)
+        OnionrCommunicatorTimers(self, self.getBlocks, self._core.config.get('timers.getBlocks'), requiresPeer=True)
         OnionrCommunicatorTimers(self, self.clearOfflinePeer, 58)
         OnionrCommunicatorTimers(self, self.daemonTools.cleanOldBlocks, 65)
         OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True)
         OnionrCommunicatorTimers(self, self.daemonTools.cooldownPeer, 30, requiresPeer=True)
+        OnionrCommunicatorTimers(self, self.uploadBlock, 10, requiresPeer=True, maxThreads=1)
         netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600)
         announceTimer = OnionrCommunicatorTimers(self, self.daemonTools.announceNode, 305, requiresPeer=True, maxThreads=1)
         cleanupTimer = OnionrCommunicatorTimers(self, self.peerCleanup, 300, requiresPeer=True)
@@ -170,7 +169,7 @@ class OnionrCommunicatorDaemon:
                 else:
                     continue
             newDBHash = self.peerAction(peer, 'getDBHash') # get their db hash
-            if newDBHash == False:
+            if newDBHash == False or not self._core._utils.validateHash(newDBHash):
                 continue # if request failed, restart loop (peer is added to offline peers automatically)
             triedPeers.append(peer)
             if newDBHash != self._core.getAddressInfo(peer, 'DBHash'):
@@ -409,7 +408,7 @@ class OnionrCommunicatorDaemon:
         '''Perform a get request to a peer'''
         if len(peer) == 0:
             return False
-        logger.info('Performing ' + action + ' with ' + peer + ' on port ' + str(self.proxyPort))
+        #logger.debug('Performing ' + action + ' with ' + peer + ' on port ' + str(self.proxyPort))
         url = 'http://' + peer + '/public/?action=' + action
         if len(data) > 0:
             url += '&data=' + data
@@ -475,8 +474,7 @@ class OnionrCommunicatorDaemon:
                     if i.timerFunction.__name__ == 'lookupAdders':
                         i.count = (i.frequency - 1)
             elif cmd[0] == 'uploadBlock':
-                self.blockToUpload = cmd[1]
+                self.blocksToUpload.append(cmd[1])
-                threading.Thread(target=self.uploadBlock).start()
             elif cmd[0] == 'startSocket':
                 # Create our own socket server
                 socketInfo = json.loads(cmd[1])
@@ -497,8 +495,12 @@ class OnionrCommunicatorDaemon:
         '''Upload our block to a few peers'''
         # when inserting a block, we try to upload it to a few peers to add some deniability
         triedPeers = []
-        if not self._core._utils.validateHash(self.blockToUpload):
+        finishedUploads = []
+        if len(self.blocksToUpload) != 0:
+            for bl in self.blocksToUpload:
+                if not self._core._utils.validateHash(bl):
                     logger.warn('Requested to upload invalid block')
+                    self.decrementThreadCount('uploadBlock')
                     return
                 for i in range(max(len(self.onlinePeers), 2)):
                     peer = self.pickOnlinePeer()
@@ -506,14 +508,23 @@ class OnionrCommunicatorDaemon:
                         continue
                     triedPeers.append(peer)
                     url = 'http://' + peer + '/public/upload/'
-            data = {'block': block.Block(self.blockToUpload).getRaw()}
+                    data = {'block': block.Block(bl).getRaw()}
                     proxyType = ''
                     if peer.endswith('.onion'):
                         proxyType = 'tor'
                     elif peer.endswith('.i2p'):
                         proxyType = 'i2p'
-            logger.info("Uploading block")
+                    logger.info("Uploading block to " + peer)
-            self._core._utils.doPostRequest(url, data=data, proxyType=proxyType)
+                    if not self._core._utils.doPostRequest(url, data=data, proxyType=proxyType) == False:
+                        self._core._utils.localCommand('waitForShare', data=bl)
+                        finishedUploads.append(bl)
+                        break
+        for x in finishedUploads:
+            try:
+                self.blocksToUpload.remove(x)
+            except ValueError:
+                pass
+        self.decrementThreadCount('uploadBlock')
 
     def announce(self, peer):
         '''Announce to peers our address'''

onionr/core.py

@@ -592,7 +592,6 @@ class Core:
         '''
 
         conn = sqlite3.connect(self.blockDB, timeout=10)
-
         c = conn.cursor()
 
         execute = 'SELECT dateReceived FROM hashes WHERE hash=?;'
@@ -609,7 +608,6 @@ class Core:
         '''
 
         conn = sqlite3.connect(self.blockDB, timeout=10)
-
         c = conn.cursor()
 
         if orderDate:
@@ -623,6 +621,7 @@ class Core:
         for row in c.execute(execute, args):
             for i in row:
                 rows.append(i)
+
         return rows
 
     def getExpiredBlocks(self):
@@ -631,10 +630,10 @@ class Core:
         c = conn.cursor()
         date = int(self._utils.getEpoch())
 
-        execute = 'SELECT hash FROM hashes WHERE expire <= ? ORDER BY dateReceived;'
+        execute = 'SELECT hash FROM hashes WHERE expire <= %s ORDER BY dateReceived;' % (date,)
 
         rows = list()
-        for row in c.execute(execute, (date,)):
+        for row in c.execute(execute):
             for i in row:
                 rows.append(i)
         return rows
@@ -680,8 +679,7 @@ class Core:
 
         return True
 
-
+    def insertBlock(self, data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = None, expire=None):
-    def insertBlock(self, data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = dict(), expire=None):
         '''
             Inserts a block into the network
             encryptType must be specified to encrypt a block
@@ -784,6 +782,8 @@ class Core:
         payload = proof.waitForResult()
         if payload != False:
             retData = self.setData(payload)
+            # Tell the api server through localCommand to wait for the daemon to upload this block to make stastical analysis more difficult
+            self._utils.localCommand('waitForShare', data=retData)
             self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
             #self.setBlockType(retData, meta['type'])
             self._utils.processBlockMetadata(retData)

onionr/onionr.py

@@ -40,7 +40,7 @@ except ImportError:
     raise Exception("You need the PySocks module (for use with socks5 proxy to use Tor)")
 
 ONIONR_TAGLINE = 'Anonymous P2P Platform - GPLv3 - https://Onionr.VoidNet.Tech'
-ONIONR_VERSION = '0.3.0' # for debugging and stuff
+ONIONR_VERSION = '0.3.2' # for debugging and stuff
 ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION)
 API_VERSION = '5' # increments of 1; only change when something fundemental about how the API works changes. This way other nodes know how to communicate without learning too much information about you.
 
@@ -64,7 +64,6 @@ class Onionr:
             self.dataDir = 'data/'
 
         # Load global configuration data
-
         data_exists = Onionr.setupConfig(self.dataDir, self = self)
 
         self.onionrCore = core.Core()
@@ -169,6 +168,7 @@ class Onionr:
 
             'add-file': self.addFile,
             'addfile': self.addFile,
+
             'get-file': self.getFile,
             'getfile': self.getFile,
 
@@ -186,6 +186,17 @@ class Onionr:
             'gui' : self.openUI,
             'chat': self.startChat,
 
+            'getpassword': self.printWebPassword,
+            'get-password': self.printWebPassword,
+            'getpwd': self.printWebPassword,
+            'get-pwd': self.printWebPassword,
+            'getpass': self.printWebPassword,
+            'get-pass': self.printWebPassword,
+            'getpasswd': self.printWebPassword,
+            'get-passwd': self.printWebPassword,
+
+            'chat': self.startChat,
+
             'friend': self.friendCmd
         }
 
@@ -246,7 +257,6 @@ class Onionr:
         for detail in details:
             logger.info('%s%s: \n%s%s\n' % (logger.colors.fg.lightgreen, detail, logger.colors.fg.green, details[detail]), sensitive = True)
 
-
     def startChat(self):
         try:
             data = json.dumps({'peer': sys.argv[2], 'reason': 'chat'})
@@ -300,6 +310,48 @@ class Onionr:
                 logger.info('Syntax: friend add/remove/list [address]')
 
 
+    def friendCmd(self):
+        '''List, add, or remove friend(s)
+        Changes their peer DB entry.
+        '''
+        friend = ''
+        try:
+            # Get the friend command
+            action = sys.argv[2]
+        except IndexError:
+            logger.info('Syntax: friend add/remove/list [address]')
+        else:
+            action = action.lower()
+            if action == 'list':
+                # List out peers marked as our friend
+                for friend in self.onionrCore.listPeers(randomOrder=False, trust=1):
+                    if friend == self.onionrCore._crypto.pubKey: # do not list our key
+                        continue
+                    friendProfile = onionrusers.OnionrUser(self.onionrCore, friend)
+                    logger.info(friend + ' - ' + friendProfile.getName())
+            elif action in ('add', 'remove'):
+                try:
+                    friend = sys.argv[3]
+                    if not self.onionrUtils.validatePubKey(friend):
+                        raise onionrexceptions.InvalidPubkey('Public key is invalid')
+                    if friend not in self.onionrCore.listPeers():
+                        raise onionrexceptions.KeyNotKnown
+                    friend = onionrusers.OnionrUser(self.onionrCore, friend)
+                except IndexError:
+                    logger.error('Friend ID is required.')
+                except onionrexceptions.KeyNotKnown:
+                    logger.error('That peer is not in our database')
+                else:
+                    if action == 'add':
+                        friend.setTrust(1)
+                        logger.info('Added %s as friend.' % (friend.publicKey,))
+                    else:
+                        friend.setTrust(0)
+                        logger.info('Removed %s as friend.' % (friend.publicKey,))
+            else:
+                logger.info('Syntax: friend add/remove/list [address]')
+
+
     def banBlock(self):
         try:
             ban = sys.argv[2]
@@ -633,6 +685,7 @@ class Onionr:
                 logger.debug('Started .onion service: %s' % (logger.colors.underline + net.myID))
                 logger.debug('Using public key: %s' % (logger.colors.underline + self.onionrCore._crypto.pubKey))
                 time.sleep(1)
+
                 # TODO: make runable on windows
                 communicatorProc = subprocess.Popen([communicatorDaemon, 'run', str(net.socksPort)])
 
@@ -798,6 +851,7 @@ class Onionr:
             logger.error("Syntax %s %s" % (sys.argv[0], '/path/to/filename <blockhash>'))
         else:
             logger.info(fileName)
+
             contents = None
             if os.path.exists(fileName):
                 logger.error("File already exists")
@@ -805,6 +859,7 @@ class Onionr:
             if not self.onionrUtils.validateHash(bHash):
                 logger.error('Block hash is invalid')
                 return
+
             Block.mergeChain(bHash, fileName)
         return
 

onionr/onionrblacklist.py

@@ -30,13 +30,16 @@ class OnionrBlackList:
     def inBlacklist(self, data):
         hashed = self._core._utils.bytesToStr(self._core._crypto.sha3Hash(data))
         retData = False
+
         if not hashed.isalnum():
             raise Exception("Hashed data is not alpha numeric")
         if len(hashed) > 64:
             raise Exception("Hashed data is too large")
+
         for i in self._dbExecute("SELECT * FROM blacklist WHERE hash = ?", (hashed,)):
             retData = True # this only executes if an entry is present by that hash
             break
+        
         return retData
 
     def _dbExecute(self, toExec, params = ()):

onionr/onionrblockapi.py

@@ -243,9 +243,9 @@ class Block:
                 if (not self.getBlockFile() is None) and (recreate is True):
                     with open(self.getBlockFile(), 'wb') as blockFile:
                         blockFile.write(self.getRaw().encode())
-                    self.update()
                 else:
                     self.hash = self.getCore().insertBlock(self.getContent(), header = self.getType(), sign = sign, meta = self.getMetadata(), expire = self.getExpire())
+                
                 self.update()
 
                 return self.getHash()

onionr/onionrchat.py

@@ -46,4 +46,5 @@ class OnionrChat:
                     self.communicator.socketClient.sendData(peer, "lol")
                 except:
                     pass
+
             time.sleep(2)

onionr/onionrdaemontools.py

@@ -17,6 +17,7 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
+
 import onionrexceptions, onionrpeers, onionrproofs, logger, onionrusers
 import base64, sqlite3, os
 from dependencies import secrets
@@ -89,6 +90,7 @@ class DaemonTools:
         c = conn.cursor()
         time = self.daemon._core._utils.getEpoch()
         deleteKeys = []
+
         for entry in c.execute("SELECT * FROM forwardKeys WHERE expire <= ?", (time,)):
             logger.info(entry[1])
             deleteKeys.append(entry[1])
@@ -120,6 +122,7 @@ class DaemonTools:
         # Cool down a peer, if we have max connections alive for long enough
         if onlinePeerAmount >= self.daemon._core.config.get('peers.max_connect', 10):
             finding = True
+
             while finding:
                 try:
                     toCool = min(tempConnectTimes, key=tempConnectTimes.get)
@@ -132,6 +135,7 @@ class DaemonTools:
             else:
                 self.daemon.removeOnlinePeer(toCool)
                 self.daemon.cooldownPeer[toCool] = self.daemon._core._utils.getEpoch()
+
         self.daemon.decrementThreadCount('cooldownPeer')
 
     def runCheck(self):

onionr/onionrsockets.py

@@ -87,6 +87,7 @@ class OnionrSocketServer:
     def detectShutdown(self):
         while not self._core.killSockets:
             time.sleep(5)
+
         logger.debug('Killing socket server...')
         self.http_server.stop()
 

onionr/onionrusers.py

@@ -107,12 +107,14 @@ class OnionrUser:
 
         conn.commit()
         conn.close()
+
         return key
 
     def _getForwardKeys(self):
         conn = sqlite3.connect(self._core.peerDB, timeout=10)
         c = conn.cursor()
         keyList = []
+
         for row in c.execute("SELECT forwardKey FROM forwardKeys WHERE peerKey = ? ORDER BY date DESC", (self.publicKey,)):
             key = row[0]
             keyList.append(key)
@@ -150,8 +152,10 @@ class OnionrUser:
         pubkey = self._core._utils.bytesToStr(pubkey)
         command = (pubkey,)
         keyList = [] # list of tuples containing pub, private for peer
+
         for result in c.execute("SELECT * FROM myForwardKeys WHERE peer = ?", command):
             keyList.append((result[1], result[2]))
+
         if len(keyList) == 0:
             if genNew:
                 self.generateForwardKey()

onionr/onionrutils.py

@@ -18,7 +18,7 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 # Misc functions that do not fit in the main api, but are useful
-import getpass, sys, requests, os, socket, hashlib, logger, sqlite3, config, binascii, time, base64, json, glob, shutil, math, json, re
+import getpass, sys, requests, os, socket, hashlib, logger, sqlite3, config, binascii, time, base64, json, glob, shutil, math, json, re, urllib.parse
 import nacl.signing, nacl.encoding
 from onionrblockapi import Block
 import onionrexceptions
@@ -150,7 +150,7 @@ class OnionrUtils:
             logger.error('Failed to read my address.', error = error)
             return None
 
-    def localCommand(self, command, silent = True):
+    def localCommand(self, command, data='', silent = True):
         '''
             Send a command to the local http API server, securely. Intended for local clients, DO NOT USE for remote peers.
         '''
@@ -164,6 +164,8 @@ class OnionrUtils:
         except FileNotFoundError:
             return False
         payload = 'http://%s:%s/client/?action=%s&token=%s&timingToken=%s' % (hostname, config.get('client.port'), command, config.get('client.hmac'), self.timingToken)
+        if data != '':
+            payload += '&data=' + urllib.parse.quote_plus(data)
         try:
             retData = requests.get(payload).text
         except Exception as error:
@@ -495,6 +497,12 @@ class OnionrUtils:
                 except binascii.Error:
                     retVal = False
 
+                # Validate address is valid base32 (when capitalized and minus extension); v2/v3 onions and .b32.i2p use base32
+                try:
+                    base64.b32decode(idNoDomain.upper().encode())
+                except binascii.Error:
+                    retVal = False
+
             return retVal
         except:
             return False

onionr/static-data/default-plugins/cliui/main.py

@@ -52,7 +52,7 @@ class OnionrCLIUI:
 
         while showMenu:
             if firstRun:
-                logger.info('Please wait while Onionr starts...'')
+                logger.info('Please wait while Onionr starts...')
                 daemon = subprocess.Popen(["./onionr.py", "start"], stdin=subprocess.PIPE, stdout=subprocess.DEVNULL)
                 time.sleep(30)
                 firstRun = False
@@ -63,7 +63,6 @@ class OnionrCLIUI:
                 isOnline = "No"
 
             logger.info('''Daemon Running: ''' + isOnline + '''
-
 1. Flow (Anonymous public chat, use at your own risk)
 2. Mail (Secure email-like service)
 3. File Sharing
@@ -91,6 +90,7 @@ class OnionrCLIUI:
                 if isOnline == "Yes":
                     logger.info("Onionr daemon will shutdown...")
                     self.myCore.daemonQueueAdd('shutdown')
+
                     try:
                         daemon.kill()
                     except UnboundLocalError:

onionr/static-data/default-plugins/encrypt/main.py

@@ -72,6 +72,7 @@ class PlainEncryption:
             encrypted = self.api.get_core()._crypto.pubKeyEncrypt(plaintext, pubkey, anonymous=True, encodedData=True)
             encrypted = self.api.get_core()._utils.bytesToStr(encrypted)
             logger.info('Encrypted Message: \n\nONIONR ENCRYPTED DATA %s END ENCRYPTED DATA' % (encrypted,))
+
     def decrypt(self):
         plaintext = ""
         data = ""
@@ -102,7 +103,6 @@ class PlainEncryption:
                 logger.info("Message has good signature.")
         return
 
-
 def on_init(api, data = None):
     '''
         This event is called after Onionr is initialized, but before the command
@@ -114,4 +114,5 @@ def on_init(api, data = None):
     encrypt = PlainEncryption(pluginapi)
     api.commands.register(['encrypt'], encrypt.encrypt)
     api.commands.register(['decrypt'], encrypt.decrypt)
+    
     return

onionr/static-data/default-plugins/pms/main.py

@@ -129,8 +129,10 @@ class OnionrMail:
                 else:
                     cancel = ''
                     readBlock.verifySig()
+
                     logger.info('Message recieved from %s' % (self.myCore._utils.bytesToStr(readBlock.signer,)))
                     logger.info('Valid signature: %s' % readBlock.validSig)
+
                     if not readBlock.validSig:
                         logger.warn('This message has an INVALID signature. ANYONE could have sent this message.')
                         cancel = logger.readline('Press enter to continue to message, or -q to not open the message (recommended).')
@@ -172,6 +174,7 @@ class OnionrMail:
         for i in self.sentboxTools.listSent():
             self.sentboxList.append(i['hash'])
             self.sentMessages[i['hash']] = (i['message'], i['peer'])
+
             logger.info('%s. %s - %s - %s' % (count, i['hash'], i['peer'][:12], i['date']))
             count += 1
 

onionr/static-data/default_config.json

@@ -2,6 +2,8 @@
     "general" : {
         "dev_mode" : true,
         "display_header" : true,
+        "minimum_block_pow": 5,
+        "minimum_send_pow": 5,
 
         "minimum_block_pow": 5,
         "minimum_send_pow": 5,