Merge remote-tracking branch 'origin/tempblocks' into onionrui

.gitignore

@@ -14,3 +14,4 @@ onionr/.onionr-lock
 core
 .vscode/*
 venv/*
+onionr/fs*

RUN-LINUX.sh

@@ -1,3 +1,4 @@
 #!/bin/sh
+cd "$(dirname "$0")"
 cd onionr/
 ./onionr.py "$@"

onionr/api.py

@@ -1,5 +1,5 @@
 '''
-    Onionr - P2P Microblogging Platform & Social network
+    Onionr - P2P Anonymous Storage Network
 
     This file handles all incoming http requests to the client, using Flask
 '''
@@ -22,7 +22,7 @@ from flask import request, Response, abort, send_from_directory
 from multiprocessing import Process
 from gevent.pywsgi import WSGIServer
 import sys, random, threading, hmac, hashlib, base64, time, math, os, json
-from core import Core
+import core
 from onionrblockapi import Block
 import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config
 
@@ -37,6 +37,9 @@ class API:
         '''
             Validate that the client token matches the given token
         '''
+        if len(self.clientToken) == 0:
+            logger.error("client password needs to be set")
+            return False
         try:
             if not hmac.compare_digest(self.clientToken, token):
                 return False
@@ -64,7 +67,7 @@ class API:
 
         return 'text/plain'
 
-    def __init__(self, debug):
+    def __init__(self, debug, API_VERSION):
         '''
             Initialize the api server, preping variables for later use
 
@@ -83,7 +86,7 @@ class API:
 
         self.debug = debug
         self._privateDelayTime = 3
-        self._core = Core()
+        self._core = core.Core()
         self._crypto = onionrcrypto.OnionrCrypto(self._core)
         self._utils = onionrutils.OnionrUtils(self._core)
         app = flask.Flask(__name__)
@@ -97,7 +100,7 @@ class API:
         self.mimeType = 'text/plain'
         self.overrideCSP = False
 
-        with open('data/time-bypass.txt', 'w') as bypass:
+        with open(self._core.dataDir + 'time-bypass.txt', 'w') as bypass:
             bypass.write(self.timeBypassToken)
 
         if not debug and not self._developmentMode:
@@ -106,7 +109,7 @@ class API:
         else:
             self.host = '127.0.0.1'
 
-        with open('data/host.txt', 'w') as file:
+        with open(self._core.dataDir + 'host.txt', 'w') as file:
             file.write(self.host)
 
         @app.before_request
@@ -128,14 +131,14 @@ class API:
                 resp.headers["Content-Security-Policy"] =  "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'"
             resp.headers['X-Frame-Options'] = 'deny'
             resp.headers['X-Content-Type-Options'] = "nosniff"
-            resp.headers['server'] = 'Onionr'
+            resp.headers['api'] = API_VERSION
 
             # reset to text/plain to help prevent browser attacks
             self.mimeType = 'text/plain'
             self.overrideCSP = False
 
             return resp
-
+            
         @app.route('/www/private/<path:path>')
         def www_private(path):
             startTime = math.floor(time.time())
@@ -477,7 +480,7 @@ class API:
             elif action == 'getData':
                 resp = ''
                 if self._utils.validateHash(data):
-                    if os.path.exists('data/blocks/' + data + '.dat'):
+                    if os.path.exists(self._core.dataDir + 'blocks/' + data + '.dat'):
                         block = Block(hash=data.encode(), core=self._core)
                         resp = base64.b64encode(block.getRaw().encode()).decode()
                 if len(resp) == 0:
@@ -526,7 +529,7 @@ class API:
             while len(self._core.hsAddress) == 0:
                 self._core.refreshFirstStartVars()
                 time.sleep(0.5)
-            self.http_server = WSGIServer((self.host, bindPort), app)
+            self.http_server = WSGIServer((self.host, bindPort), app, log=None)
             self.http_server.serve_forever()
         except KeyboardInterrupt:
             pass

onionr/communicator2.py

@@ -19,9 +19,10 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import sys, os, core, config, json, requests, time, logger, threading, base64, onionr
+import sys, os, core, config, json, requests, time, logger, threading, base64, onionr, uuid
 import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block
-import onionrdaemontools
+import onionrdaemontools, onionrsockets, onionrchat
+from dependencies import secrets
 from defusedxml import minidom
 
 class OnionrCommunicatorDaemon:
@@ -79,8 +80,10 @@ class OnionrCommunicatorDaemon:
         #self.daemonTools = onionrdaemontools.DaemonTools(self)
         self.daemonTools = onionrdaemontools.DaemonTools(self)
 
+        self._chat = onionrchat.OnionrChat(self)
+
         if debug or developmentMode:
-            OnionrCommunicatorTimers(self, self.heartbeat, 10)
+            OnionrCommunicatorTimers(self, self.heartbeat, 30)
 
         # Set timers, function reference, seconds
         # requiresPeer True means the timer function won't fire if we have no connected peers
@@ -91,17 +94,25 @@ class OnionrCommunicatorDaemon:
         OnionrCommunicatorTimers(self, self.getBlocks, self._core.config.get('timers.get_blocks'), requiresPeer=True)
         OnionrCommunicatorTimers(self, self.clearOfflinePeer, 58)
         OnionrCommunicatorTimers(self, self.daemonTools.cleanOldBlocks, 65)
-        OnionrCommunicatorTimers(self, self.lookupKeys, 60, requiresPeer=True)
         OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True)
         OnionrCommunicatorTimers(self, self.daemonTools.cooldownPeer, 30, requiresPeer=True)
         netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600)
         announceTimer = OnionrCommunicatorTimers(self, self.daemonTools.announceNode, 305, requiresPeer=True, maxThreads=1)
         cleanupTimer = OnionrCommunicatorTimers(self, self.peerCleanup, 300, requiresPeer=True)
+        forwardSecrecyTimer = OnionrCommunicatorTimers(self, self.daemonTools.cleanKeys, 15)
 
         # set loop to execute instantly to load up peer pool (replaced old pool init wait)
         peerPoolTimer.count = (peerPoolTimer.frequency - 1)
         cleanupTimer.count = (cleanupTimer.frequency - 60)
         announceTimer.count = (cleanupTimer.frequency - 60)
+        #forwardSecrecyTimer.count = (forwardSecrecyTimer.frequency - 990)
+
+        self.socketServer = threading.Thread(target=onionrsockets.OnionrSocketServer, args=(self._core,))
+        self.socketServer.start()
+        self.socketClient = onionrsockets.OnionrSocketClient(self._core)
+
+        # Loads chat messages into memory
+        threading.Thread(target=self._chat.chatHandler).start()
 
         # Main daemon loop, mainly for calling timers, don't do any complex operations here to avoid locking
         try:
@@ -116,21 +127,10 @@ class OnionrCommunicatorDaemon:
             pass
 
         logger.info('Goodbye.')
+        self._core.killSockets = True
         self._core._utils.localCommand('shutdown') # shutdown the api
         time.sleep(0.5)
 
-    def lookupKeys(self):
-        '''Lookup new keys'''
-        logger.debug('Looking up new keys...')
-        tryAmount = 1
-        for i in range(tryAmount): # amount of times to ask peers for new keys
-            # Download new key list from random online peers
-            peer = self.pickOnlinePeer()
-            newKeys = self.peerAction(peer, action='kex')
-            self._core._utils.mergeKeys(newKeys)
-        self.decrementThreadCount('lookupKeys')
-        return
-
     def lookupAdders(self):
         '''Lookup new peer addresses'''
         logger.info('LOOKING UP NEW ADDRESSES')
@@ -149,10 +149,13 @@ class OnionrCommunicatorDaemon:
         newBlocks = ''
         existingBlocks = self._core.getBlockList()
         triedPeers = [] # list of peers we've tried this time around
+        maxBacklog = 1560 # Max amount of *new* block hashes to have already in queue, to avoid memory exhaustion
         for i in range(tryAmount):
-            # check if disk allocation is used
+            if len(self.blockQueue) >= maxBacklog:
+                break
             if not self.isOnline:
                 break
+            # check if disk allocation is used
             if self._core._utils.storageCounter.isFull():
                 logger.debug('Not looking up new blocks due to maximum amount of allowed disk space used')
                 break
@@ -182,6 +185,7 @@ class OnionrCommunicatorDaemon:
                             if not i in existingBlocks:
                                 # if block does not exist on disk and is not already in block queue
                                 if i not in self.blockQueue and not self._core._blacklist.inBlacklist(i):
+                                    # TODO ensure block starts with minimum difficulty before adding to queue
                                     self.blockQueue.append(i) # add blocks to download queue
         self.decrementThreadCount('lookupBlocks')
         return
@@ -209,7 +213,7 @@ class OnionrCommunicatorDaemon:
             logger.info("Attempting to download %s..." % blockHash)
             peerUsed = self.pickOnlinePeer()
             content = self.peerAction(peerUsed, 'getData', data=blockHash) # block content from random peer (includes metadata)
-            if content != False:
+            if content != False and len(content) > 0:
                 try:
                     content = content.encode()
                 except AttributeError:
@@ -255,7 +259,10 @@ class OnionrCommunicatorDaemon:
                     onionrpeers.PeerProfiles(peerUsed, self._core).addScore(-50)
                     logger.warn('Block hash validation failed for ' + blockHash + ' got ' + tempHash)
                 if removeFromQueue:
-                    self.blockQueue.remove(blockHash) # remove from block queue both if success or false
+                    try:
+                        self.blockQueue.remove(blockHash) # remove from block queue both if success or false
+                    except ValueError:
+                        pass
             self.currentDownloading.remove(blockHash)
         self.decrementThreadCount('getBlocks')
         return
@@ -445,16 +452,15 @@ class OnionrCommunicatorDaemon:
             if cmd[0] == 'shutdown':
                 self.shutdown = True
             elif cmd[0] == 'announceNode':
-                self.announce(cmd[1])
+                if len(self.onlinePeers) > 0:
+                    self.announce(cmd[1])
+                else:
+                    logger.warn("Not introducing, since I have no connected nodes.")
             elif cmd[0] == 'runCheck':
                 logger.debug('Status check; looks good.')
-                open('data/.runcheck', 'w+').close()
+                open(self._core.dataDir + '.runcheck', 'w+').close()
             elif cmd[0] == 'connectedPeers':
                 self.printOnlinePeers()
-            elif cmd[0] == 'kex':
-                for i in self.timers:
-                    if i.timerFunction.__name__ == 'lookupKeys':
-                        i.count = (i.frequency - 1)
             elif cmd[0] == 'pex':
                 for i in self.timers:
                     if i.timerFunction.__name__ == 'lookupAdders':
@@ -462,6 +468,17 @@ class OnionrCommunicatorDaemon:
             elif cmd[0] == 'uploadBlock':
                 self.blockToUpload = cmd[1]
                 threading.Thread(target=self.uploadBlock).start()
+            elif cmd[0] == 'startSocket':
+                # Create our own socket server
+                socketInfo = json.loads(cmd[1])
+                socketInfo['id'] = uuid.uuid4()
+                self._core.startSocket = socketInfo
+            elif cmd[0] == 'addSocket':
+                # Socket server was created for us
+                socketInfo = json.loads(cmd[1])
+                peer = socketInfo['peer']
+                reason = socketInfo['reason']
+                threading.Thread(target=self.socketClient.startSocket, args=(peer, reason)).start()
             else:
                 logger.info('Recieved daemonQueue command:' + cmd[0])
 
@@ -491,9 +508,7 @@ class OnionrCommunicatorDaemon:
 
     def announce(self, peer):
         '''Announce to peers our address'''
-        if self.daemonTools.announceNode():
+        if self.daemonTools.announceNode() == False:
-            logger.info('Successfully introduced node to ' + peer)
-        else:
             logger.warn('Could not introduce node.')
 
     def detectAPICrash(self):

onionr/config.py

@@ -20,7 +20,14 @@
 
 import os, json, logger
 
-_configfile = os.path.abspath('data/config.json')
+try:
+    dataDir = os.environ['ONIONR_HOME']
+    if not dataDir.endswith('/'):
+        dataDir += '/'
+except KeyError:
+    dataDir = 'data/'
+    
+_configfile = os.path.abspath(dataDir + 'config.json')
 _config = {}
 
 def get(key, default = None):

onionr/core.py

@@ -17,11 +17,11 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import sqlite3, os, sys, time, math, base64, tarfile, getpass, simplecrypt, hashlib, nacl, logger, json, netcontroller, math, config
+import sqlite3, os, sys, time, math, base64, tarfile, nacl, logger, json, netcontroller, math, config
 from onionrblockapi import Block
 
 import onionrutils, onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions, onionrvalues
-import onionrblacklist
+import onionrblacklist, onionrchat, onionrusers
 import dbcreator
 
 if sys.version_info < (3, 6):
@@ -38,33 +38,50 @@ class Core:
         '''
 
         try:
-            self.queueDB = 'data/queue.db'
+            self.dataDir = os.environ['ONIONR_HOME']
-            self.peerDB = 'data/peers.db'
+            if not self.dataDir.endswith('/'):
-            self.blockDB = 'data/blocks.db'
+                self.dataDir += '/'
-            self.blockDataLocation = 'data/blocks/'
+        except KeyError:
-            self.addressDB = 'data/address.db'
+            self.dataDir = 'data/'
+
+        try:
+            self.queueDB = self.dataDir + 'queue.db'
+            self.peerDB = self.dataDir + 'peers.db'
+            self.blockDB = self.dataDir + 'blocks.db'
+            self.blockDataLocation = self.dataDir + 'blocks/'
+            self.addressDB = self.dataDir + 'address.db'
             self.hsAddress = ''
             self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt'
             self.bootstrapList = []
             self.requirements = onionrvalues.OnionrValues()
             self.torPort = torPort
-            self.dataNonceFile = 'data/block-nonces.dat'
+            self.dataNonceFile = self.dataDir + 'block-nonces.dat'
             self.dbCreate = dbcreator.DBCreator(self)
+            self.forwardKeysFile = self.dataDir + 'forward-keys.db'
 
-            self.usageFile = 'data/disk-usage.txt'
+            # Socket data, defined here because of multithreading constraints with gevent
+            self.killSockets = False
+            self.startSocket = {}
+            self.socketServerConnData = {}
+            self.socketReasons = {}
+            self.socketServerResponseData = {}
+
+            self.usageFile = self.dataDir + 'disk-usage.txt'
             self.config = config
 
             self.maxBlockSize = 10000000 # max block size in bytes
 
-            if not os.path.exists('data/'):
+            if not os.path.exists(self.dataDir):
-                os.mkdir('data/')
+                os.mkdir(self.dataDir)
-            if not os.path.exists('data/blocks/'):
+            if not os.path.exists(self.dataDir + 'blocks/'):
-                os.mkdir('data/blocks/')
+                os.mkdir(self.dataDir + 'blocks/')
             if not os.path.exists(self.blockDB):
                 self.createBlockDB()
+            if not os.path.exists(self.forwardKeysFile):
+                self.dbCreate.createForwardKeyDB()
 
-            if os.path.exists('data/hs/hostname'):
+            if os.path.exists(self.dataDir + '/hs/hostname'):
-                with open('data/hs/hostname', 'r') as hs:
+                with open(self.dataDir + '/hs/hostname', 'r') as hs:
                     self.hsAddress = hs.read().strip()
 
             # Load bootstrap address list
@@ -91,8 +108,9 @@ class Core:
         '''
             Hack to refresh some vars which may not be set on first start
         '''
-        if os.path.exists('data/hs/hostname'):
+
-            with open('data/hs/hostname', 'r') as hs:
+        if os.path.exists(self.dataDir + '/hs/hostname'):
+            with open(self.dataDir + '/hs/hostname', 'r') as hs:
                 self.hsAddress = hs.read().strip()
 
     def addPeer(self, peerID, powID, name=''):
@@ -107,7 +125,9 @@ class Core:
             logger.warn("POW token for pubkey base64 representation exceeded 120 bytes, is " + str(sys.getsizeof(powID)))
             return False
 
-        conn = sqlite3.connect(self.peerDB)
+        events.event('pubkey_add', data = {'key': peerID}, onionr = None)
+
+        conn = sqlite3.connect(self.peerDB, timeout=10)
         hashID = self._crypto.pubKeyHashID(peerID)
         c = conn.cursor()
         t = (peerID, name, 'unknown', hashID, powID, 0)
@@ -132,11 +152,10 @@ class Core:
             Add an address to the address database (only tor currently)
         '''
 
-        if address == config.get('i2p.own_addr', None):
+        if address == config.get('i2p.ownAddr', None) or address == self.hsAddress:
-
             return False
         if self._utils.validateID(address):
-            conn = sqlite3.connect(self.addressDB)
+            conn = sqlite3.connect(self.addressDB, timeout=10)
             c = conn.cursor()
             # check if address is in database
             # this is safe to do because the address is validated above, but we strip some chars here too just in case
@@ -169,7 +188,7 @@ class Core:
         '''
 
         if self._utils.validateID(address):
-            conn = sqlite3.connect(self.addressDB)
+            conn = sqlite3.connect(self.addressDB, timeout=10)
             c = conn.cursor()
             t = (address,)
             c.execute('Delete from adders where address=?;', t)
@@ -189,13 +208,13 @@ class Core:
         '''
 
         if self._utils.validateHash(block):
-            conn = sqlite3.connect(self.blockDB)
+            conn = sqlite3.connect(self.blockDB, timeout=10)
             c = conn.cursor()
             t = (block,)
             c.execute('Delete from hashes where hash=?;', t)
             conn.commit()
             conn.close()
-            blockFile = 'data/blocks/%s.dat' % block
+            blockFile = self.dataDir + '/blocks/%s.dat' % block
             dataSize = 0
             try:
                 ''' Get size of data when loaded as an object/var, rather than on disk,
@@ -240,7 +259,7 @@ class Core:
             raise Exception('Block db does not exist')
         if self._utils.hasBlock(newHash):
             return
-        conn = sqlite3.connect(self.blockDB)
+        conn = sqlite3.connect(self.blockDB, timeout=10)
         c = conn.cursor()
         currentTime = self._utils.getEpoch()
         if selfInsert or dataSaved:
@@ -269,14 +288,6 @@ class Core:
 
         return data
 
-    def _getSha3Hash(self, data):
-        hasher = hashlib.sha3_256()
-        if not type(data) is bytes:
-            data = data.encode()
-        hasher.update(data)
-        dataHash = hasher.hexdigest()
-        return dataHash
-
     def setData(self, data):
         '''
             Set the data assciated with a hash
@@ -288,7 +299,7 @@ class Core:
         if not type(data) is bytes:
             data = data.encode()
 
-        dataHash = self._getSha3Hash(data)
+        dataHash = self._crypto.sha3Hash(data)
 
         if type(dataHash) is bytes:
             dataHash = dataHash.decode()
@@ -301,7 +312,7 @@ class Core:
                 blockFile = open(blockFileName, 'wb')
                 blockFile.write(data)
                 blockFile.close()
-                conn = sqlite3.connect(self.blockDB)
+                conn = sqlite3.connect(self.blockDB, timeout=10)
                 c = conn.cursor()
                 c.execute("UPDATE hashes SET dataSaved=1 WHERE hash = ?;", (dataHash,))
                 conn.commit()
@@ -313,44 +324,6 @@ class Core:
 
         return dataHash
 
-    def dataDirEncrypt(self, password):
-        '''
-            Encrypt the data directory on Onionr shutdown
-        '''
-
-        if os.path.exists('data.tar'):
-            os.remove('data.tar')
-        tar = tarfile.open("data.tar", "w")
-        for name in ['data']:
-            tar.add(name)
-        tar.close()
-        tarData = open('data.tar', 'r',  encoding = 'ISO-8859-1').read()
-        encrypted = simplecrypt.encrypt(password, tarData)
-        open('data-encrypted.dat', 'wb').write(encrypted)
-        os.remove('data.tar')
-
-        return
-
-    def dataDirDecrypt(self, password):
-        '''
-            Decrypt the data directory on startup
-        '''
-
-        if not os.path.exists('data-encrypted.dat'):
-            return (False, 'encrypted archive does not exist')
-        data = open('data-encrypted.dat', 'rb').read()
-        try:
-            decrypted = simplecrypt.decrypt(password, data)
-        except simplecrypt.DecryptionException:
-            return (False, 'wrong password (or corrupted archive)')
-        else:
-            open('data.tar', 'wb').write(decrypted)
-            tar = tarfile.open('data.tar')
-            tar.extractall()
-            tar.close()
-
-        return (True, '')
-
     def daemonQueue(self):
         '''
             Gives commands to the communication proccess/daemon by reading an sqlite3 database
@@ -360,16 +333,16 @@ class Core:
 
         retData = False
         if not os.path.exists(self.queueDB):
-            self.makeDaemonDB()
+            self.dbCreate.createDaemonDB()
         else:
-            conn = sqlite3.connect(self.queueDB)
+            conn = sqlite3.connect(self.queueDB, timeout=10)
             c = conn.cursor()
             try:
                 for row in c.execute('SELECT command, data, date, min(ID) FROM commands group by id'):
                     retData = row
                     break
             except sqlite3.OperationalError:
-                self.makeDaemonDB()
+                self.dbCreate.createDaemonDB()
             else:
                 if retData != False:
                     c.execute('DELETE FROM commands WHERE id=?;', (retData[3],))
@@ -380,22 +353,6 @@ class Core:
 
         return retData
 
-    def makeDaemonDB(self):
-        '''
-            Generate the daemon queue db
-        '''
-
-        conn = sqlite3.connect(self.queueDB)
-        c = conn.cursor()
-
-        # Create table
-        c.execute('''CREATE TABLE commands
-                    (id integer primary key autoincrement, command text, data text, date text)''')
-        conn.commit()
-
-        conn.close()
-        return
-
     def daemonQueueAdd(self, command, data=''):
         '''
             Add a command to the daemon queue, used by the communication daemon (communicator.py)
@@ -405,8 +362,7 @@ class Core:
         # Intended to be used by the web server
 
         date = self._utils.getEpoch()
-        conn = sqlite3.connect(self.queueDB)
+        conn = sqlite3.connect(self.queueDB, timeout=10)
-
         c = conn.cursor()
         t = (command, data, date)
 
@@ -425,8 +381,7 @@ class Core:
         '''
             Clear the daemon queue (somewhat dangerous)
         '''
-
+        conn = sqlite3.connect(self.queueDB, timeout=10)
-        conn = sqlite3.connect(self.queueDB)
         c = conn.cursor()
 
         try:
@@ -444,7 +399,7 @@ class Core:
         '''
             Return a list of addresses
         '''
-        conn = sqlite3.connect(self.addressDB)
+        conn = sqlite3.connect(self.addressDB, timeout=10)
         c = conn.cursor()
         if randomOrder:
             addresses = c.execute('SELECT * FROM adders ORDER BY RANDOM();')
@@ -463,8 +418,7 @@ class Core:
             randomOrder determines if the list should be in a random order
             trust sets the minimum trust to list
         '''
-
+        conn = sqlite3.connect(self.peerDB, timeout=10)
-        conn = sqlite3.connect(self.peerDB)
         c = conn.cursor()
 
         payload = ''
@@ -509,21 +463,19 @@ class Core:
             id text             0
             name text,          1
             adders text,        2
-            forwardKey text,    3
+            dateSeen not null,  3
-            dateSeen not null,  4
+            bytesStored int,    4
-            bytesStored int,    5
+            trust int           5
-            trust int           6
+            pubkeyExchanged int 6
-            pubkeyExchanged int 7
+            hashID text         7
-            hashID text         8
+            pow text            8
-            pow text            9
         '''
-
+        conn = sqlite3.connect(self.peerDB, timeout=10)
-        conn = sqlite3.connect(self.peerDB)
         c = conn.cursor()
 
         command = (peer,)
 
-        infoNumbers = {'id': 0, 'name': 1, 'adders': 2, 'forwardKey': 3, 'dateSeen': 4, 'bytesStored': 5, 'trust': 6, 'pubkeyExchanged': 7, 'hashID': 8}
+        infoNumbers = {'id': 0, 'name': 1, 'adders': 2, 'dateSeen': 3, 'bytesStored': 4, 'trust': 5, 'pubkeyExchanged': 6, 'hashID': 7}
         info = infoNumbers[info]
         iterCount = 0
         retVal = ''
@@ -545,7 +497,7 @@ class Core:
             Update a peer for a key
         '''
 
-        conn = sqlite3.connect(self.peerDB)
+        conn = sqlite3.connect(self.peerDB, timeout=10)
         c = conn.cursor()
 
         command = (data, peer)
@@ -574,7 +526,7 @@ class Core:
             lastConnect 7
         '''
 
-        conn = sqlite3.connect(self.addressDB)
+        conn = sqlite3.connect(self.addressDB, timeout=10)
         c = conn.cursor()
 
         command = (address,)
@@ -599,7 +551,7 @@ class Core:
             Update an address for a key
         '''
 
-        conn = sqlite3.connect(self.addressDB)
+        conn = sqlite3.connect(self.addressDB, timeout=10)
         c = conn.cursor()
 
         command = (data, address)
@@ -619,7 +571,7 @@ class Core:
             Get list of our blocks
         '''
 
-        conn = sqlite3.connect(self.blockDB)
+        conn = sqlite3.connect(self.blockDB, timeout=10)
         c = conn.cursor()
 
         if unsaved:
@@ -639,7 +591,8 @@ class Core:
             Returns the date a block was received
         '''
 
-        conn = sqlite3.connect(self.blockDB)
+        conn = sqlite3.connect(self.blockDB, timeout=10)
+
         c = conn.cursor()
 
         execute = 'SELECT dateReceived FROM hashes WHERE hash=?;'
@@ -655,7 +608,8 @@ class Core:
             Returns a list of blocks by the type
         '''
 
-        conn = sqlite3.connect(self.blockDB)
+        conn = sqlite3.connect(self.blockDB, timeout=10)
+
         c = conn.cursor()
 
         if orderDate:
@@ -669,7 +623,20 @@ class Core:
         for row in c.execute(execute, args):
             for i in row:
                 rows.append(i)
+        return rows
 
+    def getExpiredBlocks(self):
+        '''Returns a list of expired blocks'''
+        conn = sqlite3.connect(self.blockDB, timeout=10)
+        c = conn.cursor()
+        date = int(self._utils.getEpoch())
+
+        execute = 'SELECT hash FROM hashes WHERE expire <= %s ORDER BY dateReceived;' % (date,)
+
+        rows = list()
+        for row in c.execute(execute):
+            for i in row:
+                rows.append(i)
         return rows
 
     def setBlockType(self, hash, blockType):
@@ -677,7 +644,7 @@ class Core:
             Sets the type of block
         '''
 
-        conn = sqlite3.connect(self.blockDB)
+        conn = sqlite3.connect(self.blockDB, timeout=10)
         c = conn.cursor()
         c.execute("UPDATE hashes SET dataType = ? WHERE hash = ?;", (blockType, hash))
         conn.commit()
@@ -698,12 +665,13 @@ class Core:
             sig    - optional signature by the author (not optional if author is specified)
             author       - multi-round partial sha3-256 hash of authors public key
             dateClaimed  - timestamp claimed inside the block, only as trustworthy as the block author is
+            expire       - expire date for a block
         '''
 
-        if key not in ('dateReceived', 'decrypted', 'dataType', 'dataFound', 'dataSaved', 'sig', 'author', 'dateClaimed'):
+        if key not in ('dateReceived', 'decrypted', 'dataType', 'dataFound', 'dataSaved', 'sig', 'author', 'dateClaimed', 'expire'):
             return False
 
-        conn = sqlite3.connect(self.blockDB)
+        conn = sqlite3.connect(self.blockDB, timeout=10)
         c = conn.cursor()
         args = (data, hash)
         c.execute("UPDATE hashes SET " + key + " = ? where hash = ?;", args)
@@ -712,7 +680,8 @@ class Core:
 
         return True
 
-    def insertBlock(self, data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = dict()):
+
+    def insertBlock(self, data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = dict(), expire=None):
         '''
             Inserts a block into the network
             encryptType must be specified to encrypt a block
@@ -747,8 +716,6 @@ class Core:
             meta['type'] = header
         meta['type'] = str(meta['type'])
 
-        jsonMeta = json.dumps(meta)
-
         if encryptType in ('asym', 'sym', ''):
             metadata['encryptType'] = encryptType
         else:
@@ -758,7 +725,20 @@ class Core:
             data = data.encode()
         except AttributeError:
             pass
-        # sign before encrypt, as unauthenticated crypto should not be a problem here
+
+        if encryptType == 'asym':
+            try:
+                forwardEncrypted = onionrusers.OnionrUser(self, asymPeer).forwardEncrypt(data)
+                data = forwardEncrypted[0]
+                meta['forwardEnc'] = True
+            except onionrexceptions.InvalidPubkey:
+                onionrusers.OnionrUser(self, asymPeer).generateForwardKey()
+            else:
+                logger.info(forwardEncrypted)
+            onionrusers.OnionrUser(self, asymPeer).generateForwardKey()
+            fsKey = onionrusers.OnionrUser(self, asymPeer).getGeneratedForwardKeys()[0]
+            meta['newFSKey'] = fsKey[0]
+        jsonMeta = json.dumps(meta)
         if sign:
             signature = self._crypto.edSign(jsonMeta.encode() + data, key=self._crypto.privKey, encodeResult=True)
             signer = self._crypto.pubKey
@@ -766,8 +746,11 @@ class Core:
         if len(jsonMeta) > 1000:
             raise onionrexceptions.InvalidMetadata('meta in json encoded form must not exceed 1000 bytes')
 
+        user = onionrusers.OnionrUser(self, symKey)
+
         # encrypt block metadata/sig/content
         if encryptType == 'sym':
+
             if len(symKey) < self.requirements.passwordLength:
                 raise onionrexceptions.SecurityError('Weak encryption key')
             jsonMeta = self._crypto.symmetricEncrypt(jsonMeta, key=symKey, returnEncoded=True).decode()
@@ -776,6 +759,8 @@ class Core:
             signer = self._crypto.symmetricEncrypt(signer, key=symKey, returnEncoded=True).decode()
         elif encryptType == 'asym':
             if self._utils.validatePubKey(asymPeer):
+                # Encrypt block data with forward secrecy key first, but not meta
+                jsonMeta = json.dumps(meta)
                 jsonMeta = self._crypto.pubKeyEncrypt(jsonMeta, asymPeer, encodedData=True, anonymous=True).decode()
                 data = self._crypto.pubKeyEncrypt(data, asymPeer, encodedData=True, anonymous=True).decode()
                 signature = self._crypto.pubKeyEncrypt(signature, asymPeer, encodedData=True, anonymous=True).decode()
@@ -789,13 +774,19 @@ class Core:
         metadata['signer'] = signer
         metadata['time'] = str(self._utils.getEpoch())
 
+        # ensure expire is integer and of sane length
+        if type(expire) is not type(None):
+            assert len(str(int(expire))) < 14
+            metadata['expire'] = expire
+
         # send block data (and metadata) to POW module to get tokenized block data
         proof = onionrproofs.POW(metadata, data)
         payload = proof.waitForResult()
         if payload != False:
             retData = self.setData(payload)
             self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
-            self.setBlockType(retData, meta['type'])
+            #self.setBlockType(retData, meta['type'])
+            self._utils.processBlockMetadata(retData)
             self.daemonQueueAdd('uploadBlock', retData)
 
         if retData != False:

onionr/dbcreator.py

@@ -61,7 +61,6 @@ class DBCreator:
             ID text not null,
             name text,
             adders text,
-            forwardKey text,
             dateSeen not null,
             bytesStored int,
             trust int,
@@ -69,6 +68,12 @@ class DBCreator:
             hashID text,
             pow text not null);
         ''')
+        c.execute('''CREATE TABLE forwardKeys(
+        peerKey text not null,
+        forwardKey text not null,
+        date int not null,
+        expire int not null
+        );''')
         conn.commit()
         conn.close()
         return
@@ -86,6 +91,7 @@ class DBCreator:
             sig    - optional signature by the author (not optional if author is specified)
             author       - multi-round partial sha3-256 hash of authors public key
             dateClaimed  - timestamp claimed inside the block, only as trustworthy as the block author is
+            expire int   - block expire date in epoch
         '''
         if os.path.exists(self.core.blockDB):
             raise Exception("Block database already exists")
@@ -100,9 +106,42 @@ class DBCreator:
             dataSaved int,
             sig text,
             author text,
-            dateClaimed int
+            dateClaimed int,
+            expire int
             );
         ''')
         conn.commit()
         conn.close()
-        return
+        return
+
+    def createForwardKeyDB(self):
+        '''
+            Create the forward secrecy key db (*for *OUR* keys*)
+        '''
+        if os.path.exists(self.core.forwardKeysFile):
+            raise Exception("Block database already exists")
+        conn = sqlite3.connect(self.core.forwardKeysFile)
+        c = conn.cursor()
+        c.execute('''CREATE TABLE myForwardKeys(
+            peer text not null,
+            publickey text not null,
+            privatekey text not null,
+            date int not null,
+            expire int not null
+            );
+        ''')
+        conn.commit()
+        conn.close()
+        return
+    
+    def createDaemonDB(self):
+        '''
+            Create the daemon queue database
+        '''
+        conn = sqlite3.connect(self.core.queueDB, timeout=10)
+        c = conn.cursor()
+        # Create table
+        c.execute('''CREATE TABLE commands
+                    (id integer primary key autoincrement, command text, data text, date text)''')
+        conn.commit()
+        conn.close()

onionr/netcontroller.py

@@ -18,21 +18,37 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 
-import subprocess, os, random, sys, logger, time, signal, config
+import subprocess, os, random, sys, logger, time, signal, config, base64
+from stem.control import Controller
 from onionrblockapi import Block
-
+from dependencies import secrets
 class NetController:
     '''
         This class handles hidden service setup on Tor and I2P
     '''
 
     def __init__(self, hsPort):
-        self.torConfigLocation = 'data/torrc'
+        try:
+            self.dataDir = os.environ['ONIONR_HOME']
+            if not self.dataDir.endswith('/'):
+                self.dataDir += '/'
+        except KeyError:
+            self.dataDir = 'data/'
+
+        self.torConfigLocation = self.dataDir + 'torrc'
         self.readyState = False
         self.socksPort = random.randint(1024, 65535)
         self.hsPort = hsPort
         self._torInstnace = ''
         self.myID = ''
+
+        if os.path.exists('./tor'):
+            self.torBinary = './tor'
+        elif os.path.exists('/usr/bin/tor'):
+            self.torBinary = '/usr/bin/tor'
+        else:
+            self.torBinary = 'tor'
+
         config.reload()
         '''
             if os.path.exists(self.torConfigLocation):
@@ -52,13 +68,33 @@ class NetController:
         if config.get('tor.v3_onions'):
             hsVer = 'HiddenServiceVersion 3'
             logger.info('Using v3 onions :)')
+
         if os.path.exists(self.torConfigLocation):
             os.remove(self.torConfigLocation)
+
+        # Set the Tor control password. Meant to make it harder to manipulate our Tor instance
+        plaintext = base64.b64encode(os.urandom(50)).decode()
+        config.set('tor.controlpassword', plaintext, savefile=True)
+        config.set('tor.socksport', self.socksPort, savefile=True)
+
+        controlPort = random.randint(1025, 65535)
+
+        config.set('tor.controlPort', controlPort, savefile=True)
+
+        hashedPassword = subprocess.Popen([self.torBinary, '--hash-password', plaintext], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        for line in iter(hashedPassword.stdout.readline, b''):
+            password = line.decode()
+            if 'warn' not in password:
+                break
+
         torrcData = '''SocksPort ''' + str(self.socksPort) + '''
-HiddenServiceDir data/hs/
+HiddenServiceDir ''' + self.dataDir + '''hs/
 \n''' + hsVer + '''\n
 HiddenServicePort 80 127.0.0.1:''' + str(self.hsPort) + '''
-DataDirectory data/tordata/
+DataDirectory ''' + self.dataDir + '''tordata/
+CookieAuthentication 1
+ControlPort ''' + str(controlPort) + '''
+HashedControlPassword ''' + str(password) + '''
         '''
         torrc = open(self.torConfigLocation, 'w')
         torrc.write(torrcData)
@@ -74,20 +110,20 @@ DataDirectory data/tordata/
         self.generateTorrc()
 
         if os.path.exists('./tor'):
-            torBinary = './tor'
+            self.torBinary = './tor'
         elif os.path.exists('/usr/bin/tor'):
-            torBinary = '/usr/bin/tor'
+            self.torBinary = '/usr/bin/tor'
         else:
-            torBinary = 'tor'
+            self.torBinary = 'tor'
 
         try:
-            tor = subprocess.Popen([torBinary, '-f', self.torConfigLocation], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            tor = subprocess.Popen([self.torBinary, '-f', self.torConfigLocation], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         except FileNotFoundError:
             logger.fatal("Tor was not found in your path or the Onionr directory. Please install Tor and try again.")
             sys.exit(1)
         else:
             # Test Tor Version
-            torVersion = subprocess.Popen([torBinary, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            torVersion = subprocess.Popen([self.torBinary, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
             for line in iter(torVersion.stdout.readline, b''):
                 if 'Tor 0.2.' in line.decode():
                     logger.warn("Running 0.2.x Tor series, no support for v3 onion peers")
@@ -111,11 +147,11 @@ DataDirectory data/tordata/
         logger.debug('Finished starting Tor.', timestamp=True)
         self.readyState = True
 
-        myID = open('data/hs/hostname', 'r')
+        myID = open(self.dataDir + 'hs/hostname', 'r')
         self.myID = myID.read().replace('\n', '')
         myID.close()
 
-        torPidFile = open('data/torPid.txt', 'w')
+        torPidFile = open(self.dataDir + 'torPid.txt', 'w')
         torPidFile.write(str(tor.pid))
         torPidFile.close()
 
@@ -127,7 +163,7 @@ DataDirectory data/tordata/
         '''
 
         try:
-            pid = open('data/torPid.txt', 'r')
+            pid = open(self.dataDir + 'torPid.txt', 'r')
             pidN = pid.read()
             pid.close()
         except FileNotFoundError:
@@ -140,7 +176,7 @@ DataDirectory data/tordata/
 
         try:
             os.kill(int(pidN), signal.SIGTERM)
-            os.remove('data/torPid.txt')
+            os.remove(self.dataDir + 'torPid.txt')
         except ProcessLookupError:
             pass
         except FileNotFoundError:

onionr/onionr.py

@@ -26,10 +26,10 @@ if sys.version_info[0] == 2 or sys.version_info[1] < 5:
     print('Error, Onionr requires Python 3.4+')
     sys.exit(1)
 import os, base64, random, getpass, shutil, subprocess, requests, time, platform, datetime, re, json, getpass, sqlite3
+import webbrowser
 from threading import Thread
 import api, core, config, logger, onionrplugins as plugins, onionrevents as events
 import onionrutils
-from onionrutils import OnionrUtils
 from netcontroller import NetController
 from onionrblockapi import Block
 import onionrproofs, onionrexceptions, onionrusers
@@ -42,7 +42,7 @@ except ImportError:
 ONIONR_TAGLINE = 'Anonymous P2P Platform - GPLv3 - https://Onionr.VoidNet.Tech'
 ONIONR_VERSION = '0.3.0' # for debugging and stuff
 ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION)
-API_VERSION = '4' # increments of 1; only change when something fundemental about how the API works changes. This way other nodes know how to communicate without learning too much information about you.
+API_VERSION = '5' # increments of 1; only change when something fundemental about how the API works changes. This way other nodes know how to communicate without learning too much information about you.
 
 class Onionr:
     def __init__(self):
@@ -50,23 +50,31 @@ class Onionr:
             Main Onionr class. This is for the CLI program, and does not handle much of the logic.
             In general, external programs and plugins should not use this class.
         '''
+        self.userRunDir = os.getcwd() # Directory user runs the program from
         try:
             os.chdir(sys.path[0])
         except FileNotFoundError:
             pass
 
+        try:
+            self.dataDir = os.environ['ONIONR_HOME']
+            if not self.dataDir.endswith('/'):
+                self.dataDir += '/'
+        except KeyError:
+            self.dataDir = 'data/'
+
         # Load global configuration data
 
-        data_exists = os.path.exists('data/')
+        data_exists = os.path.exists(self.dataDir)
 
         if not data_exists:
-            os.mkdir('data/')
+            os.mkdir(self.dataDir)
 
         if os.path.exists('static-data/default_config.json'):
             config.set_config(json.loads(open('static-data/default_config.json').read())) # this is the default config, it will be overwritten if a config file already exists. Else, it saves it
         else:
             # the default config file doesn't exist, try hardcoded config
-            config.set_config({'dev_mode': True, 'log': {'file': {'output': True, 'path': 'data/output.log'}, 'console': {'output': True, 'color': True}}})
+            config.set_config({'dev_mode': True, 'log': {'file': {'output': True, 'path': self.dataDir + 'output.log'}, 'console': {'output': True, 'color': True}}})
         if not data_exists:
             config.save()
         config.reload() # this will read the configuration file into memory
@@ -78,7 +86,7 @@ class Onionr:
             settings = settings | logger.OUTPUT_TO_CONSOLE
         if config.get('log.file.output', True):
             settings = settings | logger.OUTPUT_TO_FILE
-            logger.set_file(config.get('log.file.path', '/tmp/onionr.log'))
+            logger.set_file(config.get('log.file.path', '/tmp/onionr.log').replace('data/', self.dataDir))
         logger.set_settings(settings)
 
         if str(config.get('general.dev_mode', True)).lower() == 'true':
@@ -89,37 +97,27 @@ class Onionr:
             logger.set_level(logger.LEVEL_INFO)
 
         self.onionrCore = core.Core()
-        self.onionrUtils = OnionrUtils(self.onionrCore)
+        self.onionrUtils = onionrutils.OnionrUtils(self.onionrCore)
 
         # Handle commands
 
         self.debug = False # Whole application debugging
 
-        if os.path.exists('data-encrypted.dat'):
+        # If data folder does not exist
-            while True:
+        if not data_exists:
-                print('Enter password to decrypt:')
+            if not os.path.exists(self.dataDir + 'blocks/'):
-                password = getpass.getpass()
+                os.mkdir(self.dataDir + 'blocks/')
-                result = self.onionrCore.dataDirDecrypt(password)
-                if os.path.exists('data/'):
-                    break
-                else:
-                    logger.error('Failed to decrypt: ' + result[1], timestamp = False)
-        else:
-            # If data folder does not exist
-            if not data_exists:
-                if not os.path.exists('data/blocks/'):
-                    os.mkdir('data/blocks/')
 
-            # Copy default plugins into plugins folder
+        # Copy default plugins into plugins folder
-            if not os.path.exists(plugins.get_plugins_folder()):
+        if not os.path.exists(plugins.get_plugins_folder()):
-                if os.path.exists('static-data/default-plugins/'):
+            if os.path.exists('static-data/default-plugins/'):
-                    names = [f for f in os.listdir("static-data/default-plugins/") if not os.path.isfile(f)]
+                names = [f for f in os.listdir("static-data/default-plugins/") if not os.path.isfile(f)]
-                    shutil.copytree('static-data/default-plugins/', plugins.get_plugins_folder())
+                shutil.copytree('static-data/default-plugins/', plugins.get_plugins_folder())
 
-                    # Enable plugins
+                # Enable plugins
-                    for name in names:
+                for name in names:
-                        if not name in plugins.get_enabled_plugins():
+                    if not name in plugins.get_enabled_plugins():
-                            plugins.enable(name, self)
+                        plugins.enable(name, self)
 
         for name in plugins.get_enabled_plugins():
             if not os.path.exists(plugins.get_plugin_data_folder(name)):
@@ -190,6 +188,10 @@ class Onionr:
 
             'add-file': self.addFile,
             'addfile': self.addFile,
+
+            'get-file': self.getFile,
+            'getfile': self.getFile,
+
             'listconn': self.listConn,
 
             'import-blocks': self.onionrUtils.importNewBlocks,
@@ -212,6 +214,8 @@ class Onionr:
             'getpasswd': self.printWebPassword,
             'get-passwd': self.printWebPassword,
 
+            'chat': self.startChat,
+
             'friend': self.friendCmd
         }
 
@@ -230,6 +234,7 @@ class Onionr:
             'add-peer': 'Adds a peer to database',
             'list-peers': 'Displays a list of peers',
             'add-file': 'Create an Onionr block from a file',
+            'get-file': 'Get a file from Onionr blocks',
             'import-blocks': 'import blocks from the disk (Onionr is transport-agnostic!)',
             'listconn': 'list connected peers',
             'kex': 'exchange keys with peers (done automatically)',
@@ -250,17 +255,20 @@ class Onionr:
         finally:
             self.execute(command)
 
-        if not self._developmentMode:
-            encryptionPassword = self.onionrUtils.getPassword('Enter password to encrypt directory: ')
-            self.onionrCore.dataDirEncrypt(encryptionPassword)
-            shutil.rmtree('data/')
-
         return
 
     '''
         THIS SECTION HANDLES THE COMMANDS
     '''
 
+    def startChat(self):
+        try:
+            data = json.dumps({'peer': sys.argv[2], 'reason': 'chat'})
+        except IndexError:
+            logger.error('Must specify peer to chat with.')
+        else:
+            self.onionrCore.daemonQueueAdd('startSocket', data)
+
     def getCommands(self):
         return self.cmds
 
@@ -612,7 +620,7 @@ class Onionr:
         '''
         communicatorDaemon = './communicator2.py'
 
-        apiThread = Thread(target=api.API, args=(self.debug,))
+        apiThread = Thread(target=api.API, args=(self.debug,API_VERSION))
         apiThread.start()
         try:
             time.sleep(3)
@@ -632,7 +640,7 @@ class Onionr:
                 logger.info('Our Public key: ' + self.onionrCore._crypto.pubKey)
                 time.sleep(1)
                 #TODO make runable on windows
-                subprocess.Popen([communicatorDaemon, "run", str(net.socksPort)])
+                communicatorProc = subprocess.Popen([communicatorDaemon, "run", str(net.socksPort)])
                 # Print nice header thing :)
                 if config.get('general.display_header', True):
                     self.header()
@@ -641,6 +649,9 @@ class Onionr:
                 try:
                     while True:
                         time.sleep(5)
+                        # Break if communicator process ends, so we don't have left over processes
+                        if communicatorProc.poll() is not None:
+                            break
                 except KeyboardInterrupt:
                     self.onionrCore.daemonQueueAdd('shutdown')
                     self.onionrUtils.localCommand('shutdown')
@@ -675,26 +686,23 @@ class Onionr:
             # define stats messages here
             totalBlocks = len(Block.getBlocks())
             signedBlocks = len(Block.getBlocks(signed = True))
-            powToken = self.onionrCore._crypto.pubKeyPowToken
             messages = {
                 # info about local client
-                'Onionr Daemon Status' : ((logger.colors.fg.green + 'Online') if self.onionrUtils.isCommunicatorRunning(timeout = 2) else logger.colors.fg.red + 'Offline'),
+                'Onionr Daemon Status' : ((logger.colors.fg.green + 'Online') if self.onionrUtils.isCommunicatorRunning(timeout = 9) else logger.colors.fg.red + 'Offline'),
                 'Public Key' : self.onionrCore._crypto.pubKey,
-                'POW Token' : powToken,
-                'Combined' : self.onionrCore._crypto.pubKey + '-' + powToken,
                 'Human readable public key' : self.onionrCore._utils.getHumanReadableID(),
                 'Node Address' : self.get_hostname(),
 
                 # file and folder size stats
                 'div1' : True, # this creates a solid line across the screen, a div
-                'Total Block Size' : onionrutils.humanSize(onionrutils.size('data/blocks/')),
+                'Total Block Size' : onionrutils.humanSize(onionrutils.size(self.dataDir + 'blocks/')),
-                'Total Plugin Size' : onionrutils.humanSize(onionrutils.size('data/plugins/')),
+                'Total Plugin Size' : onionrutils.humanSize(onionrutils.size(self.dataDir + 'plugins/')),
-                'Log File Size' : onionrutils.humanSize(onionrutils.size('data/output.log')),
+                'Log File Size' : onionrutils.humanSize(onionrutils.size(self.dataDir + 'output.log')),
 
                 # count stats
                 'div2' : True,
                 'Known Peers Count' : str(len(self.onionrCore.listPeers()) - 1),
-                'Enabled Plugins Count' : str(len(config.get('plugins.enabled', list()))) + ' / ' + str(len(os.listdir('data/plugins/'))),
+                'Enabled Plugins Count' : str(len(config.get('plugins.enabled', list()))) + ' / ' + str(len(os.listdir(self.dataDir + 'plugins/'))),
                 'Known Blocks Count' : str(totalBlocks),
                 'Percent Blocks Signed' : str(round(100 * signedBlocks / max(totalBlocks, 1), 2)) + '%'
             }
@@ -760,7 +768,7 @@ class Onionr:
 
     def get_hostname(self):
         try:
-            with open('./data/hs/hostname', 'r') as hostname:
+            with open('./' + self.dataDir + 'hs/hostname', 'r') as hostname:
                 return hostname.read().strip()
         except Exception:
             return None
@@ -780,6 +788,27 @@ class Onionr:
 
         return columns
 
+    def getFile(self):
+        '''
+            Get a file from onionr blocks
+        '''
+        try:
+            fileName = sys.argv[2]
+            bHash = sys.argv[3]
+        except IndexError:
+            logger.error("Syntax %s %s" % (sys.argv[0], '/path/to/filename <blockhash>'))
+        else:
+            print(fileName)
+            contents = None
+            if os.path.exists(fileName):
+                logger.error("File already exists")
+                return
+            if not self.onionrUtils.validateHash(bHash):
+                logger.error('Block hash is invalid')
+                return
+            Block.mergeChain(bHash, fileName)
+        return
+
     def addFile(self):
         '''
             Adds a file to the onionr network
@@ -790,8 +819,9 @@ class Onionr:
             contents = None
 
             if not os.path.exists(filename):
-                logger.warn('That file does not exist. Improper path?')
+                logger.error('That file does not exist. Improper path (specify full path)?')
-
+                return
+            logger.info('Adding file... this might take a long time.')
             try:
                 blockhash = Block.createChain(file = filename)
                 logger.info('File %s saved in block %s.' % (filename, blockhash))
@@ -801,7 +831,6 @@ class Onionr:
             logger.error('%s add-file <filename>' % sys.argv[0], timestamp = False)
 
     def openUI(self):
-        import webbrowser
         url = 'http://127.0.0.1:%s/ui/index.html?timingToken=%s' % (config.get('client.port', 59496), self.onionrUtils.getTimeBypassToken())
 
         print('Opening %s ...' % url)

onionr/onionrblacklist.py

@@ -20,7 +20,7 @@
 import sqlite3, os, logger
 class OnionrBlackList:
     def __init__(self, coreInst):
-        self.blacklistDB = 'data/blacklist.db'
+        self.blacklistDB = coreInst.dataDir + 'blacklist.db'
         self._core = coreInst
         
         if not os.path.exists(self.blacklistDB):
@@ -32,7 +32,8 @@ class OnionrBlackList:
         retData = False
         if not hashed.isalnum():
             raise Exception("Hashed data is not alpha numeric")
-
+        if len(hashed) > 64:
+            raise Exception("Hashed data is too large")
         for i in self._dbExecute("select * from blacklist where hash='%s'" % (hashed,)):
             retData = True # this only executes if an entry is present by that hash
             break
@@ -95,9 +96,8 @@ class OnionrBlackList:
         '''
         # we hash the data so we can remove data entirely from our node's disk
         hashed = self._core._utils.bytesToStr(self._core._crypto.sha3Hash(data))
-
+        if len(hashed) > 64:
-        if self.inBlacklist(hashed):
+            raise Exception("Hashed data is too large")
-            return
 
         if not hashed.isalnum():
             raise Exception("Hashed data is not alpha numeric")
@@ -109,7 +109,8 @@ class OnionrBlackList:
             int(expire)
         except ValueError:
             raise Exception("expire is not int")
-        #TODO check for length sanity
+        if self.inBlacklist(hashed):
+            return
         insert = (hashed,)
         blacklistDate = self._core._utils.getEpoch()
         self._dbExecute("insert into blacklist (hash, dataType, blacklistDate, expire) VALUES('%s', %s, %s, %s);" % (hashed, dataType, blacklistDate, expire))

onionr/onionrblockapi.py

@@ -18,14 +18,14 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 
-import core as onionrcore, logger, config, onionrexceptions, nacl.exceptions
+import core as onionrcore, logger, config, onionrexceptions, nacl.exceptions, onionrusers
 import json, os, sys, datetime, base64
 
 class Block:
     blockCacheOrder = list() # NEVER write your own code that writes to this!
     blockCache = dict() # should never be accessed directly, look at Block.getCache()
 
-    def __init__(self, hash = None, core = None, type = None, content = None):
+    def __init__(self, hash = None, core = None, type = None, content = None, expire=None):
         # take from arguments
         # sometimes people input a bytes object instead of str in `hash`
         if (not hash is None) and isinstance(hash, bytes):
@@ -35,6 +35,7 @@ class Block:
         self.core = core
         self.btype = type
         self.bcontent = content
+        self.expire = expire
 
         # initialize variables
         self.valid = True
@@ -91,9 +92,18 @@ class Block:
                 self.signature = core._crypto.pubKeyDecrypt(self.signature, anonymous=anonymous, encodedData=encodedData)
                 self.signer = core._crypto.pubKeyDecrypt(self.signer, anonymous=anonymous, encodedData=encodedData)
                 self.signedData =  json.dumps(self.bmetadata) + self.bcontent.decode()
+                try:
+                    assert self.bmetadata['forwardEnc'] is True
+                except (AssertionError, KeyError) as e:
+                    pass
+                else:
+                    try:
+                        self.bcontent = onionrusers.OnionrUser(self.getCore(), self.signer).forwardDecrypt(self.bcontent)
+                    except (onionrexceptions.DecryptionError, nacl.exceptions.CryptoError) as e:
+                        logger.error(str(e))
+                        pass
             except nacl.exceptions.CryptoError:
-                pass
+                logger.debug('Could not decrypt block. Either invalid key or corrupted data')
-                #logger.debug('Could not decrypt block. Either invalid key or corrupted data')
             else:
                 retData = True
                 self.decrypted = True
@@ -151,7 +161,7 @@ class Block:
 
                     # read from file if it's still None
                     if blockdata is None:
-                        filelocation = 'data/blocks/%s.dat' % self.getHash()
+                        filelocation = self.core.dataDir + 'blocks/%s.dat' % self.getHash()
 
                 if readfile:
                     with open(filelocation, 'rb') as f:
@@ -179,6 +189,7 @@ class Block:
             # signed data is jsonMeta + block content (no linebreak)
             self.signedData = (None if not self.isSigned() else self.getHeader('meta') + self.getContent())
             self.date = self.getCore().getBlockDate(self.getHash())
+            self.claimedTime = self.getHeader('time', None)
 
             if not self.getDate() is None:
                 self.date = datetime.datetime.fromtimestamp(self.getDate())
@@ -234,7 +245,7 @@ class Block:
                         blockFile.write(self.getRaw().encode())
                     self.update()
                 else:
-                    self.hash = self.getCore().insertBlock(self.getContent(), header = self.getType(), sign = sign, meta = self.getMetadata())
+                    self.hash = self.getCore().insertBlock(self.getContent(), header = self.getType(), sign = sign, meta = self.getMetadata(), expire = self.getExpire())
                     self.update()
 
                 return self.getHash()
@@ -247,6 +258,15 @@ class Block:
 
     # getters
 
+    def getExpire(self):
+        '''
+            Returns the expire time for a block
+
+            Outputs:
+            - (int): the expire time for a block, or None
+        '''
+        return self.expire
+
     def getHash(self):
         '''
             Returns the hash of the block if saved to file
@@ -734,7 +754,7 @@ class Block:
         if type(hash) == Block:
             blockfile = hash.getBlockFile()
         else:
-            blockfile = 'data/blocks/%s.dat' % hash
+            blockfile = onionrcore.Core().dataDir + 'blocks/%s.dat' % hash
 
         return os.path.exists(blockfile) and os.path.isfile(blockfile)
 

onionr/onionrchat.py

@@ -0,0 +1,49 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    Onionr Chat Messages
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import logger, time
+
+class OnionrChat:
+    def __init__(self, communicatorInst):
+        '''OnionrChat uses onionrsockets (handled by the communicator) to exchange direct chat messages'''
+        self.communicator = communicatorInst
+        self._core = self.communicator._core
+        self._utils = self._core._utils
+
+        self.chats = {} # {'peer': {'date': date, message': message}}
+        self.chatSend = {}
+
+    def chatHandler(self):
+        while not self.communicator.shutdown:
+            for peer in self._core.socketServerConnData:
+                try:
+                    assert self._core.socketReasons[peer] == "chat"
+                except (AssertionError, KeyError) as e:
+                    logger.warn('Peer is not for chat')
+                    continue
+                else:
+                    self.chats[peer] = {'date': self._core.socketServerConnData[peer]['date'], 'data': self._core.socketServerConnData[peer]['data']}
+                    logger.info("CHAT MESSAGE RECIEVED: %s" % self.chats[peer]['data'])
+            for peer in self.communicator.socketClient.sockets:
+                try:
+                    logger.info(self.communicator.socketClient.connPool[peer]['data'])
+                    self.communicator.socketClient.sendData(peer, "lol")
+                except:
+                    pass
+            time.sleep(2)

onionr/onionrcrypto.py

@@ -1,5 +1,5 @@
 '''
-    Onionr - P2P Microblogging Platform & Social network
+    Onionr - P2P Anonymous Storage Network
 
     This file handles Onionr's cryptography.
 '''
@@ -24,51 +24,32 @@ if sys.version_info[0] == 3 and sys.version_info[1] < 6:
     from dependencies import secrets
 elif sys.version_info[0] == 3 and sys.version_info[1] >= 6:
     import secrets
+import config
 
 class OnionrCrypto:
     def __init__(self, coreInstance):
+        config.reload()
         self._core = coreInstance
-        self._keyFile = 'data/keys.txt'
+        self._keyFile = self._core.dataDir + 'keys.txt'
-        self.keyPowFile = 'data/keyPow.txt'
         self.pubKey = None
         self.privKey = None
 
         self.secrets = secrets
-
+        
-        self.pubKeyPowToken = None
-        #self.pubKeyPowHash = None
-
         self.HASH_ID_ROUNDS = 2000
 
         # Load our own pub/priv Ed25519 keys, gen & save them if they don't exist
         if os.path.exists(self._keyFile):
-            with open('data/keys.txt', 'r') as keys:
+            with open(self._core.dataDir + 'keys.txt', 'r') as keys:
                 keys = keys.read().split(',')
                 self.pubKey = keys[0]
                 self.privKey = keys[1]
-            try:
-                with open(self.keyPowFile, 'r') as powFile:
-                    data = powFile.read()
-                    self.pubKeyPowToken = data
-            except (FileNotFoundError, IndexError):
-                pass
         else:
             keys = self.generatePubKey()
             self.pubKey = keys[0]
             self.privKey = keys[1]
             with open(self._keyFile, 'w') as keyfile:
                 keyfile.write(self.pubKey + ',' + self.privKey)
-            with open(self.keyPowFile, 'w') as keyPowFile:
-                proof = onionrproofs.DataPOW(self.pubKey)
-                logger.info('Doing necessary work to insert our public key')
-                while True:
-                    time.sleep(0.2)
-                    powToken = proof.getResult()
-                    if powToken != False:
-                        break
-                keyPowFile.write(base64.b64encode(powToken[1]).decode())
-                self.pubKeyPowToken = powToken[1]
-                self.pubKeyPowHash = powToken[0]
         return
 
     def edVerify(self, data, key, sig, encodedData=True):
@@ -76,7 +57,10 @@ class OnionrCrypto:
         try:
             key = nacl.signing.VerifyKey(key=key, encoder=nacl.encoding.Base32Encoder)
         except nacl.exceptions.ValueError:
-            logger.warn('Signature by unknown key (cannot reverse hash)')
+            #logger.debug('Signature by unknown key (cannot reverse hash)')
+            return False
+        except binascii.Error:
+            logger.warn('Could not load key for verification, invalid padding')
             return False
         retData = False
         sig = base64.b64decode(sig)
@@ -125,7 +109,7 @@ class OnionrCrypto:
             encoding = nacl.encoding.RawEncoder
 
         if self.privKey != None and not anonymous:
-            ownKey = nacl.signing.SigningKey(seed=self.privKey, encoder=nacl.encoding.Base32Encoder)
+            ownKey = nacl.signing.SigningKey(seed=self.privKey, encoder=nacl.encoding.Base32Encoder).to_curve25519_private_key()
             key = nacl.signing.VerifyKey(key=pubkey, encoder=nacl.encoding.Base32Encoder).to_curve25519_public_key()
             ourBox = nacl.public.Box(ownKey, key)
             retVal = ourBox.encrypt(data.encode(), encoder=encoding)
@@ -139,9 +123,9 @@ class OnionrCrypto:
             retVal = anonBox.encrypt(data, encoder=encoding)
         return retVal
 
-    def pubKeyDecrypt(self, data, pubkey='', anonymous=False, encodedData=False):
+    def pubKeyDecrypt(self, data, pubkey='', privkey='', anonymous=False, encodedData=False):
         '''pubkey decrypt (Curve25519, taken from Ed25519 pubkey)'''
-        retVal = False
+        decrypted = False
         if encodedData:
             encoding = nacl.encoding.Base64Encoder
         else:
@@ -151,7 +135,11 @@ class OnionrCrypto:
             ourBox = nacl.public.Box(ownKey, pubkey)
             decrypted = ourBox.decrypt(data, encoder=encoding)
         elif anonymous:
-            anonBox = nacl.public.SealedBox(ownKey)
+            if self._core._utils.validatePubKey(privkey):
+                privkey = nacl.signing.SigningKey(seed=privkey, encoder=nacl.encoding.Base32Encoder()).to_curve25519_private_key()
+                anonBox = nacl.public.SealedBox(privkey)
+            else:
+                anonBox = nacl.public.SealedBox(ownKey)
             decrypted = anonBox.decrypt(data, encoder=encoding)
         return decrypted
 
@@ -262,7 +250,8 @@ class OnionrCrypto:
             pass
 
         difficulty = math.floor(dataLen / 1000000)
-
+        if difficulty < int(config.get('general.minimum_block_pow')):
+            difficulty = int(config.get('general.minimum_block_pow'))
         mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode()
         puzzle = mainHash[:difficulty]
 

onionr/onionrdaemontools.py

@@ -17,7 +17,7 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import onionrexceptions, onionrpeers, onionrproofs, base64, logger
+import onionrexceptions, onionrpeers, onionrproofs, base64, logger, onionrusers, sqlite3
 from dependencies import secrets
 class DaemonTools:
     def __init__(self, daemon):
@@ -52,27 +52,55 @@ class DaemonTools:
 
         logger.info('Announcing node to ' + url)
         if self.daemon._core._utils.doPostRequest(url, data) == 'Success':
+            logger.info('Successfully introduced node to ' + peer)
             retData = True
         self.daemon.decrementThreadCount('announceNode')
         return retData
 
     def netCheck(self):
         '''Check if we are connected to the internet or not when we can't connect to any peers'''
-        if len(self.daemon.onlinePeers) != 0:
+        if len(self.daemon.onlinePeers) == 0:
             if not self.daemon._core._utils.checkNetwork(torPort=self.daemon.proxyPort):
                 logger.warn('Network check failed, are you connected to the internet?')
                 self.daemon.isOnline = False
         self.daemon.decrementThreadCount('netCheck')
 
     def cleanOldBlocks(self):
-        '''Delete old blocks if our disk allocation is full/near full'''
+        '''Delete old blocks if our disk allocation is full/near full, and also expired blocks'''
+
         while self.daemon._core._utils.storageCounter.isFull():
             oldest = self.daemon._core.getBlockList()[0]
             self.daemon._core._blacklist.addToDB(oldest)
             self.daemon._core.removeBlock(oldest)
             logger.info('Deleted block: %s' % (oldest,))
+
+        # Delete expired blocks
+        for bHash in self.daemon._core.getExpiredBlocks():
+            self.daemon._core._blacklist.addToDB(bHash)
+            self.daemon._core.removeBlock(bHash)
+        
         self.daemon.decrementThreadCount('cleanOldBlocks')
 
+    def cleanKeys(self):
+        '''Delete expired forward secrecy keys'''
+        conn = sqlite3.connect(self.daemon._core.peerDB, timeout=10)
+        c = conn.cursor()
+        time = self.daemon._core._utils.getEpoch()
+        deleteKeys = []
+        for entry in c.execute("SELECT * FROM forwardKeys where expire <= ?", (time,)):
+            logger.info(entry[1])
+            deleteKeys.append(entry[1])
+
+        for key in deleteKeys:
+            logger.info('Deleting forward key '+ key)
+            c.execute("DELETE from forwardKeys where forwardKey = ?", (key,))
+        conn.commit()
+        conn.close()
+
+        onionrusers.deleteExpiredKeys(self.daemon._core)
+
+        self.daemon.decrementThreadCount('cleanKeys')
+
     def cooldownPeer(self):
         '''Randomly add an online peer to cooldown, so we can connect a new one'''
         onlinePeerAmount = len(self.daemon.onlinePeers)

onionr/onionrexceptions.py

@@ -37,6 +37,9 @@ class InvalidPubkey(Exception):
 class KeyNotKnown(Exception):
     pass
 
+class DecryptionError(Exception):
+    pass
+
 # block exceptions
 class InvalidMetadata(Exception):
     pass
@@ -62,7 +65,15 @@ class MissingPort(Exception):
 class InvalidAddress(Exception):
     pass
 
+class InvalidAPIVersion(Exception):
+    pass
+
 # file exceptions
 
 class DiskAllocationReached(Exception):
+    pass
+
+# onionrsocket exceptions
+
+class MissingAddress(Exception):
     pass

onionr/onionri2p.py

@@ -1,19 +0,0 @@
-'''
-    Onionr - P2P Microblogging Platform & Social network
-
-    Funcitons for talking to I2P
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''

onionr/onionrplugins.py

@@ -21,7 +21,14 @@
 import os, re, importlib, config, logger
 import onionrevents as events
 
-_pluginsfolder = 'data/plugins/'
+try:
+    dataDir = os.environ['ONIONR_HOME']
+    if not dataDir.endswith('/'):
+        dataDir += '/'
+except KeyError:
+    dataDir = 'data/'
+
+_pluginsfolder = dataDir + 'plugins/'
 _instances = dict()
 
 def reload(onionr = None, stop_event = True):
@@ -217,7 +224,7 @@ def get_plugin_data_folder(name, absolute = True):
         Returns the location of a plugin's data folder
     '''
 
-    return get_plugins_folder(name, absolute) + 'data/'
+    return get_plugins_folder(name, absolute) + dataDir
 
 def check():
     '''

onionr/onionrproofs.py

@@ -1,5 +1,5 @@
 '''
-    Onionr - P2P Microblogging Platform & Social network
+    Onionr - P2P Anonymous Storage Network
 
     Proof of work module
 '''
@@ -19,7 +19,30 @@
 '''
 
 import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, logger, sys, base64, json
-import core
+import core, config
+
+def getHashDifficulty(h):
+    '''
+        Return the amount of leading zeroes in a hex hash string (h)
+    '''
+    difficulty = 0
+    assert type(h) is str
+    for character in h:
+        if character == '0':
+            difficulty += 1
+    return difficulty
+
+def hashMeetsDifficulty(h):
+    '''
+        Return bool for a hash string to see if it meets pow difficulty defined in config
+    '''
+    config.reload()
+    hashDifficulty = getHashDifficulty(h)
+    expected = int(config.get('minimum_block_pow'))
+    if hashDifficulty >= expected:
+        return True
+    else:
+        return False
 
 class DataPOW:
     def __init__(self, data, forceDifficulty=0, threadCount = 5):
@@ -27,6 +50,7 @@ class DataPOW:
         self.difficulty = 0
         self.data = data
         self.threadCount = threadCount
+        config.reload()
 
         if forceDifficulty == 0:
             dataLen = sys.getsizeof(data)
@@ -77,7 +101,6 @@ class DataPOW:
             endTime = math.floor(time.time())
             if self.reporting:
                 logger.debug('Found token after %s seconds: %s' % (endTime - startTime, token), timestamp=True)
-                logger.debug('Random value was: %s' % base64.b64encode(rand).decode())
             self.result = (token, rand)
 
     def shutdown(self):
@@ -128,7 +151,7 @@ class POW:
         dataLen = len(data) + len(json.dumps(metadata))
         self.difficulty = math.floor(dataLen / 1000000)
         if self.difficulty <= 2:
-            self.difficulty = 4
+            self.difficulty = int(config.get('general.minimum_block_pow'))
 
         try:
             self.data = self.data.encode()
@@ -144,7 +167,7 @@ class POW:
         for i in range(max(1, threadCount)):
             t = threading.Thread(name = 'thread%s' % i, target = self.pow, args = (True,myCore))
             t.start()
-        
+        self.myCore = myCore
         return
 
     def pow(self, reporting = False, myCore = None):
@@ -177,7 +200,6 @@ class POW:
             endTime = math.floor(time.time())
             if self.reporting:
                 logger.debug('Found token after %s seconds: %s' % (endTime - startTime, token), timestamp=True)
-                logger.debug('Random value was: %s' % base64.b64encode(rand).decode())
 
     def shutdown(self):
         self.hashing = False

onionr/onionrsockets.py

@@ -0,0 +1,171 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    Onionr Socket interface
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import stem.control
+import threading
+import socks, config, uuid
+import onionrexceptions, time, requests, onionrblockapi, logger
+from dependencies import secrets
+from gevent.pywsgi import WSGIServer
+from flask import request, Response, abort
+import flask
+class OnionrSocketServer:
+    def __init__(self, coreInst):
+        self._core = coreInst
+        app = flask.Flask(__name__)
+        self._core.socketServerConnData = {}
+        self.bindPort = 0
+
+        self.sockets = {}
+
+        while self.bindPort < 1024:
+            self.bindPort = secrets.randbelow(65535)
+
+        self.responseData = {}
+
+        threading.Thread(target=self.detectShutdown).start()
+        threading.Thread(target=self.socketStarter).start()
+        app = flask.Flask(__name__)
+        self.http_server = WSGIServer(('127.0.0.1', self.bindPort), app)
+        self.http_server.serve_forever()
+
+        @app.route('/dc/', methods=['POST'])
+        def acceptConn(self):
+            data = request.form['data']
+            data = self._core._utils.bytesTorStr(data)
+            data = {'date': self._core._utils.getEpoch(), 'data': data}
+            myPeer = ''
+            retData = ''
+            for peer in self.sockets:
+                if self.sockets[peer] == request.host:
+                    myPeer = peer
+                    break
+            else:
+                return ""
+
+            if request.host in self.sockets:
+                self._core.socketServerConnData[myPeer].append(data)
+            else:
+                self._core.socketServerConnData[myPeer] = [data]
+
+            try:
+                retData = self._core.socketServerResponseData[myPeer]
+            except KeyError:
+                pass
+            else:
+                self._core.socketServerResponseData[myPeer] = ''
+
+            return retData
+    
+    def socketStarter(self):
+        while not self._core.killSockets:
+            try:
+                self.addSocket(self._core.startSocket['peer'], reason=self._core.startSocket['reason'])
+            except KeyError:
+                pass
+            else:
+                logger.info('%s socket started with %s' % (self._core.startSocket['reason'], self._core.startSocket['peer']))
+                self._core.startSocket = {}
+            time.sleep(1)
+
+    def detectShutdown(self):
+        while not self._core.killSockets:
+            time.sleep(5)
+        logger.info('Killing socket server')
+        self.http_server.stop()
+
+    def addSocket(self, peer, reason=''):
+        bindPort = 1337
+
+        assert len(reason) <= 12
+            
+        with stem.control.Controller.from_port(port=config.get('tor.controlPort')) as controller:
+            controller.authenticate(config.get('tor.controlpassword'))
+
+            socket = controller.create_ephemeral_hidden_service({80: bindPort}, await_publication = True)
+            self.sockets[peer] = socket.service_id + '.onion'
+
+            self.responseData[socket.service_id + '.onion'] = ''
+
+            self._core.insertBlock(str(uuid.uuid4()), header='socket', sign=True, encryptType='asym', asymPeer=peer, meta={'reason': reason, 'address': socket.service_id + '.onion'})
+            self._core.socketReasons[peer] = reason
+        return
+    
+class OnionrSocketClient:
+    def __init__(self, coreInst):
+        self.sockets = {} # pubkey: tor address
+        self.connPool = {}
+        self.sendData = {}
+        self._core = coreInst
+        self.response = ''
+        self.request = ''
+        self.connected = False
+        self.killSocket = False
+
+    def startSocket(self, peer, reason):
+        address = ''
+        logger.info('Trying to find socket server for %s' % (peer,))
+        # Find the newest open socket for a given peer
+        for block in self._core.getBlocksByType('socket'):
+            block = onionrblockapi.Block(block, core=self._core)
+            if block.decrypt():
+                theSigner = block.signer
+                try:
+                    theSigner = theSigner.decode()
+                except AttributeError:
+                    pass
+                if block.verifySig() and theSigner == peer:
+                    address = block.getMetadata('address')
+                    if self._core._utils.validateID(address):
+                        # If we got their address, it is valid, and verified, we can break out
+                        if block.getMetadata('reason') == reason:
+                            break
+                        else:
+                            logger.error('The socket the peer opened is not for %s' % (reason,))
+                    else:
+                        logger.error('Peer transport id is invalid for socket: %s' % (address,))
+                        address = ''
+                else:
+                    logger.warn('Block has invalid sig or id, was for %s' % (theSigner,))
+        if address != '':
+            logger.info('%s socket client started with %s' % (reason, peer))
+            self.sockets[peer] = address
+            data = 'hey'
+            while not self.killSocket:
+                try:
+                    data = self.sendData[peer]
+                    logger.info('Sending %s to %s' % (data, peer))
+                except KeyError:
+                    pass
+                else:
+                    self.sendData[peer] = ''
+                postData = {'data': data}
+                self.connPool[peer] = {'date': self._core._utils.getEpoch(), 'data': self._core._utils.doPostRequest('http://' + address + '/dc/', data=postData)}
+                time.sleep(2)
+    
+    def getResponse(self, peer):
+        retData = ''
+        try:
+            retData = self.connPool[peer]
+        except KeyError:
+            pass
+        return
+    
+    def sendData(self, peer, data):
+        self.sendData[peer] = data

onionr/onionrusers.py

@@ -17,7 +17,21 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import onionrblockapi, logger, onionrexceptions
+import onionrblockapi, logger, onionrexceptions, json, sqlite3
+import nacl.exceptions
+
+def deleteExpiredKeys(coreInst):
+    # Fetch the keys we generated for the peer, that are still around
+    conn = sqlite3.connect(coreInst.forwardKeysFile, timeout=10)
+    c = conn.cursor()
+
+    curTime = coreInst._utils.getEpoch()
+    c.execute("DELETE from myForwardKeys where expire <= ?", (curTime,))
+    conn.commit()
+    conn.execute("VACUUM")
+    conn.close()
+    return
+
 class OnionrUser:
     def __init__(self, coreInst, publicKey):
         self.trust = 0
@@ -50,14 +64,114 @@ class OnionrUser:
         encrypted = coreInst._crypto.pubKeyEncrypt(data, self.publicKey, encodedData=True)
         return encrypted
     
-    def decrypt(self, data):
+    def decrypt(self, data, anonymous=True):
         decrypted = coreInst._crypto.pubKeyDecrypt(data, self.publicKey, encodedData=True)
         return decrypted
     
     def forwardEncrypt(self, data):
-        return
+        retData = ''
+        forwardKey = self._getLatestForwardKey()
+        #logger.info('using ' + forwardKey)
+        if self._core._utils.validatePubKey(forwardKey):
+            retData = self._core._crypto.pubKeyEncrypt(data, forwardKey, encodedData=True, anonymous=True)
+        else:
+            raise onionrexceptions.InvalidPubkey("No valid forward key available for this user")
+        #self.generateForwardKey()
+        return (retData, forwardKey)
     
     def forwardDecrypt(self, encrypted):
+        retData = ""
+        #logger.error(self.publicKey)
+        #logger.error(self.getGeneratedForwardKeys(False))
+        for key in self.getGeneratedForwardKeys(False):
+            logger.info(encrypted)
+            try:
+                retData = self._core._crypto.pubKeyDecrypt(encrypted, privkey=key[1], anonymous=True, encodedData=True)
+            except nacl.exceptions.CryptoError:
+                retData = False
+            else:
+                break
+        else:
+            raise onionrexceptions.DecryptionError("Could not decrypt forward secrecy content")
+        return retData
+
+    def _getLatestForwardKey(self):
+        # Get the latest forward secrecy key for a peer
+        key = ""
+        conn = sqlite3.connect(self._core.peerDB, timeout=10)
+        c = conn.cursor()
+
+        for row in c.execute("SELECT forwardKey FROM forwardKeys WHERE peerKey = ? order by date desc", (self.publicKey,)):
+            key = row[0]
+            break
+
+        conn.commit()
+        conn.close()
+        return key
+    
+    def _getForwardKeys(self):
+        conn = sqlite3.connect(self._core.peerDB, timeout=10)
+        c = conn.cursor()
+        keyList = []
+        for row in c.execute("SELECT forwardKey FROM forwardKeys WHERE peerKey = ? order by date desc", (self.publicKey,)):
+            key = row[0]
+            keyList.append(key)
+
+        conn.commit()
+        conn.close()
+
+        return list(keyList)
+
+    def generateForwardKey(self, expire=604800):
+
+        # Generate a forward secrecy key for the peer
+        conn = sqlite3.connect(self._core.forwardKeysFile, timeout=10)
+        c = conn.cursor()
+        # Prepare the insert
+        time = self._core._utils.getEpoch()
+        newKeys = self._core._crypto.generatePubKey()
+        newPub = self._core._utils.bytesToStr(newKeys[0])
+        newPriv = self._core._utils.bytesToStr(newKeys[1])
+
+        time = self._core._utils.getEpoch()
+        command = (self.publicKey, newPub, newPriv, time, expire + time)
+
+        c.execute("INSERT INTO myForwardKeys VALUES(?, ?, ?, ?, ?);", command)
+
+        conn.commit()
+        conn.close()
+        return newPub
+
+    def getGeneratedForwardKeys(self, genNew=True):
+        # Fetch the keys we generated for the peer, that are still around
+        conn = sqlite3.connect(self._core.forwardKeysFile, timeout=10)
+        c = conn.cursor()
+        pubkey = self.publicKey
+        pubkey = self._core._utils.bytesToStr(pubkey)
+        command = (pubkey,)
+        keyList = [] # list of tuples containing pub, private for peer
+        for result in c.execute("SELECT * FROM myForwardKeys where peer=?", command):
+            keyList.append((result[1], result[2]))
+        if len(keyList) == 0:
+            if genNew:
+                self.generateForwardKey()
+                keyList = self.getGeneratedForwardKeys()
+        return list(keyList)
+
+    def addForwardKey(self, newKey, expire=604800):
+        if not self._core._utils.validatePubKey(newKey):
+            raise onionrexceptions.InvalidPubkey
+        # Add a forward secrecy key for the peer
+        conn = sqlite3.connect(self._core.peerDB, timeout=10)
+        c = conn.cursor()
+        # Prepare the insert
+        time = self._core._utils.getEpoch()
+        command = (self.publicKey, newKey, time, time + expire)
+
+        c.execute("INSERT INTO forwardKeys VALUES(?, ?, ?, ?);", command)
+
+        conn.commit()
+        conn.close()
         return
     
     def findAndSetID(self):

onionr/onionrutils.py

@@ -22,7 +22,9 @@ import getpass, sys, requests, os, socket, hashlib, logger, sqlite3, config, bin
 import nacl.signing, nacl.encoding
 from onionrblockapi import Block
 import onionrexceptions
+from onionr import API_VERSION
 from defusedxml import minidom
+import onionrevents
 import pgpwords, onionrusers, storagecounter
 if sys.version_info < (3, 6):
     try:
@@ -36,20 +38,23 @@ class OnionrUtils:
         Various useful functions for validating things, etc functions, connectivity
     '''
     def __init__(self, coreInstance):
-        self.fingerprintFile = 'data/own-fingerprint.txt'
+        #self.fingerprintFile = 'data/own-fingerprint.txt' #TODO Remove since probably not needed
-        self._core = coreInstance
+        self._core = coreInstance # onionr core instance
 
-        self.timingToken = ''
+        self.timingToken = '' # for when we make local connections to our http api, to bypass timing attack defense mechanism
         self.avoidDupe = [] # list used to prevent duplicate requests per peer for certain actions
         self.peerProcessing = {} # dict of current peer actions: peer, actionList
-        self.storageCounter = storagecounter.StorageCounter(self._core)
+        self.storageCounter = storagecounter.StorageCounter(self._core) # used to keep track of how much data onionr is using on disk
-        config.reload()
+        config.reload() # onionr config
         return
 
     def getTimeBypassToken(self):
+        '''
+            Load our timingToken from disk for faster local HTTP API
+        '''
         try:
-            if os.path.exists('data/time-bypass.txt'):
+            if os.path.exists(self._core.dataDir + 'time-bypass.txt'):
-                with open('data/time-bypass.txt', 'r') as bypass:
+                with open(self._core.dataDir + 'time-bypass.txt', 'r') as bypass:
                     self.timingToken = bypass.read()
         except Exception as error:
             logger.error('Failed to fetch time bypass token.', error = error)
@@ -63,22 +68,6 @@ class OnionrUtils:
         epoch = self.getEpoch()
         return epoch - (epoch % roundS)
 
-    def incrementAddressSuccess(self, address):
-        '''
-            Increase the recorded sucesses for an address
-        '''
-        increment = self._core.getAddressInfo(address, 'success') + 1
-        self._core.setAddressInfo(address, 'success', increment)
-        return
-
-    def decrementAddressSuccess(self, address):
-        '''
-            Decrease the recorded sucesses for an address
-        '''
-        increment = self._core.getAddressInfo(address, 'success') - 1
-        self._core.setAddressInfo(address, 'success', increment)
-        return
-
     def mergeKeys(self, newKeyList):
         '''
             Merge ed25519 key list to our database, comma seperated string
@@ -88,6 +77,7 @@ class OnionrUtils:
             if newKeyList != False:
                 for key in newKeyList.split(','):
                     key = key.split('-')
+                    # Test if key is valid
                     try:
                         if len(key[0]) > 60 or len(key[1]) > 1000:
                             logger.warn('%s or its pow value is too large.' % key[0])
@@ -99,15 +89,19 @@ class OnionrUtils:
                         value = base64.b64decode(key[1])
                     except binascii.Error:
                         continue
+                    # Load the pow token
                     hashedKey = self._core._crypto.blake2bHash(key[0])
                     powHash = self._core._crypto.blake2bHash(value + hashedKey)
                     try:
                         powHash = powHash.encode()
                     except AttributeError:
                         pass
+                    # if POW meets required difficulty, TODO make configurable/dynamic
                     if powHash.startswith(b'0000'):
+                        # if we don't already have the key and its not our key, add it.
                         if not key[0] in self._core.listPeers(randomOrder=False) and type(key) != None and key[0] != self._core._crypto.pubKey:
                             if self._core.addPeer(key[0], key[1]):
+                                # Check if the peer has a set username already
                                 onionrusers.OnionrUser(self._core, key[0]).findAndSetID()
                                 retVal = True
                             else:
@@ -150,7 +144,7 @@ class OnionrUtils:
 
     def getMyAddress(self):
         try:
-            with open('./data/hs/hostname', 'r') as hostname:
+            with open('./' + self._core.dataDir + 'hs/hostname', 'r') as hostname:
                 return hostname.read().strip()
         except Exception as error:
             logger.error('Failed to read my address.', error = error)
@@ -165,7 +159,7 @@ class OnionrUtils:
         self.getTimeBypassToken()
         # TODO: URL encode parameters, just as an extra measure. May not be needed, but should be added regardless.
         try:
-            with open('data/host.txt', 'r') as host:
+            with open(self._core.dataDir + 'host.txt', 'r') as host:
                 hostname = host.read()
         except FileNotFoundError:
             return False
@@ -269,28 +263,38 @@ class OnionrUtils:
         '''
         myBlock = Block(blockHash, self._core)
         if myBlock.isEncrypted:
-            myBlock.decrypt()
+            #pass
-        blockType = myBlock.getMetadata('type') # we would use myBlock.getType() here, but it is bugged with encrypted blocks
+            logger.warn(myBlock.decrypt())
-        signer = self.bytesToStr(myBlock.signer)
+        if (myBlock.isEncrypted and myBlock.decrypted) or (not myBlock.isEncrypted):
-        try:
+            blockType = myBlock.getMetadata('type') # we would use myBlock.getType() here, but it is bugged with encrypted blocks
-            if len(blockType) <= 10:
+            signer = self.bytesToStr(myBlock.signer)
-                self._core.updateBlockInfo(blockHash, 'dataType', blockType)
+            valid = myBlock.verifySig()
 
-                if blockType == 'userInfo':
+            logger.info('Checking for fs key')
-                    if myBlock.verifySig():
+            if myBlock.getMetadata('newFSKey') is not None:
-                        peerName = myBlock.getMetadata('name')
+                onionrusers.OnionrUser(self._core, signer).addForwardKey(myBlock.getMetadata('newFSKey'))
-                        try:
+            else:
-                            if len(peerName) > 20:
+                logger.warn('FS not used for this encrypted block')
-                                raise onionrexceptions.InvalidMetdata('Peer name specified is too large')
+                logger.info(myBlock.bmetadata)
-                        except TypeError:
+        
-                            pass
+            try:
-                        except onionrexceptions.InvalidMetadata:
+                if len(blockType) <= 10:
-                            pass
+                    self._core.updateBlockInfo(blockHash, 'dataType', blockType)
-                        else:
+                    onionrevents.event('processblocks', data = {'block': myBlock, 'type': blockType, 'signer': signer, 'validSig': valid}, onionr = None)
-                            self._core.setPeerInfo(signer, 'name', peerName)
+            except TypeError:
-                            logger.info('%s is now using the name %s.' % (signer, self.escapeAnsi(peerName)))
+                logger.warn("Missing block information")
-        except TypeError:
+                pass
-            pass
+            # Set block expire time if specified
+            try:
+                expireTime = myBlock.getHeader('expire')
+                assert len(str(int(expireTime))) < 20 # test that expire time is an integer of sane length (for epoch)
+            except (AssertionError, ValueError, TypeError) as e:
+                pass
+            else:
+                self._core.updateBlockInfo(blockHash, 'expire', expireTime)
+        else:
+            logger.info(myBlock.isEncrypted)
+            logger.debug('Not processing metadata on encrypted block we cannot decrypt.')
 
     def escapeAnsi(self, line):
         '''
@@ -379,15 +383,26 @@ class OnionrUtils:
                     logger.warn('Block has invalid metadata key ' + i)
                     break
                 else:
-                    if self._core.requirements.blockMetadataLengths[i] < len(metadata[i]):
+                    testData = metadata[i]
+                    try:
+                        testData = len(testData)
+                    except (TypeError, AttributeError) as e:
+                        testData = len(str(testData))
+                    if self._core.requirements.blockMetadataLengths[i] < testData:
                         logger.warn('Block metadata key ' + i + ' exceeded maximum size')
                         break
                 if i == 'time':
                     if not self.isIntegerString(metadata[i]):
                         logger.warn('Block metadata time stamp is not integer string')
                         break
+                elif i == 'expire':
+                    try:
+                        assert int(metadata[i]) > self.getEpoch()
+                    except AssertionError:
+                        logger.warn('Block is expired')
+                        break
             else:
-                # if metadata loop gets no errors, it does not break, therefore metadata is valid
+                # if metadata loop gets no errors, it does not break, therefore metadata is valid      
                 # make sure we do not have another block with the same data content (prevent data duplication and replay attacks)
                 nonce = self._core._utils.bytesToStr(self._core._crypto.sha3Hash(blockData))
                 try:
@@ -473,6 +488,12 @@ class OnionrUtils:
                         retVal = False
                 if not idNoDomain.isalnum():
                     retVal = False
+                
+                # Validate address is valid base32 (when capitalized and minus extension); v2/v3 onions and .b32.i2p use base32
+                try:
+                    base64.b32decode(idNoDomain.upper().encode())
+                except binascii.Error:
+                    retVal = False
 
             return retVal
         except:
@@ -496,7 +517,7 @@ class OnionrUtils:
 
     def isCommunicatorRunning(self, timeout = 5, interval = 0.1):
         try:
-            runcheck_file = 'data/.runcheck'
+            runcheck_file = self._core.dataDir + '.runcheck'
 
             if os.path.isfile(runcheck_file):
                 os.remove(runcheck_file)
@@ -539,6 +560,7 @@ class OnionrUtils:
                     if self._core._crypto.sha3Hash(newBlock.read()) == block.replace('.dat', ''):
                         self._core.addToBlockDB(block.replace('.dat', ''), dataSaved=True)
                         logger.info('Imported block %s.' % block)
+                        self._core._utils.processBlockMetadata(block)
                     else:
                         logger.warn('Failed to verify hash for %s' % block)
 
@@ -604,13 +626,22 @@ class OnionrUtils:
         try:
             proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
             r = requests.get(url, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30))
+            # Check server is using same API version as us
+            try:
+                if r.headers['api'] != str(API_VERSION):
+                    raise onionrexceptions.InvalidAPIVersion
+            except KeyError:
+                raise onionrexceptions.InvalidAPIVersion
             retData = r.text
         except KeyboardInterrupt:
             raise KeyboardInterrupt
         except ValueError as e:
             logger.debug('Failed to make request', error = e)
+        except onionrexceptions.InvalidAPIVersion:
+            logger.debug("Node is using different API version :(")
         except requests.exceptions.RequestException as e:
-            logger.debug('Error: %s' % str(e))
+            if not 'ConnectTimeoutError' in str(e) and not 'Request rejected or failed' in str(e):
+                logger.debug('Error: %s' % str(e))
             retData = False
         return retData
 

onionr/onionrvalues.py

@@ -21,4 +21,4 @@
 class OnionrValues:
     def __init__(self):
         self.passwordLength = 20
-        self.blockMetadataLengths = {'meta': 1000, 'sig': 200, 'signer': 200, 'time': 10, 'powRandomToken': 1000, 'encryptType': 4} #TODO properly refine values to minimum needed
+        self.blockMetadataLengths = {'meta': 1000, 'sig': 200, 'signer': 200, 'time': 10, 'powRandomToken': 1000, 'encryptType': 4, 'expire': 14} #TODO properly refine values to minimum needed

onionr/static-data/default-plugins/cliui/main.py

@@ -46,14 +46,15 @@ class OnionrCLIUI:
         showMenu = True
         isOnline = "No"
         firstRun = True
+        choice = ''
 
         if self.myCore._utils.localCommand('ping') == 'pong':
             firstRun = False
 
         while showMenu:
             if firstRun:
-                print("please wait while Onionr starts...")
+                logger.info("please wait while Onionr starts...")
-                daemon = subprocess.Popen(["./onionr.py", "start"], stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
+                daemon = subprocess.Popen(["./onionr.py", "start"], stdin=subprocess.PIPE, stdout=subprocess.DEVNULL)
                 time.sleep(30)
                 firstRun = False
 
@@ -91,7 +92,6 @@ Daemon Running: ''' + isOnline + '''
             elif choice in ("5", "daemon"):
                 if isOnline == "Yes":
                     print("Onionr daemon will shutdown...")
-                    #self.myCore._utils.localCommand("shutdown")
                     self.myCore.daemonQueueAdd('shutdown')
                     try:
                         daemon.kill()

onionr/static-data/default-plugins/encrypt/info.json

@@ -0,0 +1,5 @@
+{
+    "name" : "encrypt",
+    "version" : "1.0",
+    "author" : "onionr"
+}

onionr/static-data/default-plugins/encrypt/main.py

@@ -0,0 +1,117 @@
+'''
+    Onionr - P2P Microblogging Platform & Social network
+
+    This default plugin allows users to encrypt/decrypt messages without using blocks
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+# Imports some useful libraries
+import logger, config, threading, time, readline, datetime, sys, json
+from onionrblockapi import Block
+import onionrexceptions, onionrusers
+import locale
+locale.setlocale(locale.LC_ALL, '')
+
+class PlainEncryption:
+    def __init__(self, api):
+        self.api = api
+        return
+    def encrypt(self):
+        # peer, data
+        plaintext = ""
+        encrypted = ""
+        # detect if signing is enabled
+        sign = True
+        try:
+            if sys.argv[3].lower() == 'false':
+                sign = False
+        except IndexError:
+            pass
+
+        try:
+            if not self.api.get_core()._utils.validatePubKey(sys.argv[2]):
+                raise onionrexceptions.InvalidPubkey
+        except (ValueError, IndexError) as e:
+            logger.error("Peer public key not specified")
+        except onionrexceptions.InvalidPubkey:
+            logger.error("Invalid public key")
+        else:
+            pubkey = sys.argv[2]
+            # Encrypt if public key is valid
+            logger.info("Please enter your message (ctrl-d or -q to stop):")
+            try:
+                for line in sys.stdin:
+                    if line == '-q\n':
+                        break
+                    plaintext += line
+            except KeyboardInterrupt:
+                sys.exit(1)
+            # Build Message to encrypt
+            data = {}
+            myPub = self.api.get_core()._crypto.pubKey
+            if sign:
+                data['sig'] = self.api.get_core()._crypto.edSign(plaintext, key=self.api.get_core()._crypto.privKey, encodeResult=True)
+                data['sig'] = self.api.get_core()._utils.bytesToStr(data['sig'])
+                data['signer'] = myPub
+            data['data'] = plaintext
+            data = json.dumps(data)
+            plaintext = data
+            encrypted = self.api.get_core()._crypto.pubKeyEncrypt(plaintext, pubkey, anonymous=True, encodedData=True)
+            encrypted = self.api.get_core()._utils.bytesToStr(encrypted)
+            print('ONIONR ENCRYPTED DATA %s END ENCRYPTED DATA' % (encrypted,))
+    def decrypt(self):
+        plaintext = ""
+        data = ""
+        logger.info("Please enter your message (ctrl-d or -q to stop):")
+        try:
+            for line in sys.stdin:
+                if line == '-q\n':
+                        break
+                data += line
+        except KeyboardInterrupt:
+            sys.exit(1)
+        if len(data) <= 1:
+            return
+        encrypted = data.replace('ONIONR ENCRYPTED DATA ', '').replace('END ENCRYPTED DATA', '')
+        myPub = self.api.get_core()._crypto.pubKey
+        decrypted = self.api.get_core()._crypto.pubKeyDecrypt(encrypted, privkey=self.api.get_core()._crypto.privKey, anonymous=True, encodedData=True)
+        if decrypted == False:
+            print("Decryption failed")
+        else:
+            data = json.loads(decrypted)
+            print(data['data'])
+            try:
+                logger.info("Signing public key: %s" % (data['signer'],))
+                assert self.api.get_core()._crypto.edVerify(data['data'], data['signer'], data['sig']) != False
+            except (AssertionError, KeyError) as e:
+                logger.warn("WARNING: THIS MESSAGE HAS A MISSING OR INVALID SIGNATURE")
+            else:
+                logger.info("Message has good signature.")
+        return
+    
+
+def on_init(api, data = None):
+    '''
+        This event is called after Onionr is initialized, but before the command
+        inputted is executed. Could be called when daemon is starting or when
+        just the client is running.
+    '''
+
+    pluginapi = api
+    encrypt = PlainEncryption(pluginapi)
+    api.commands.register(['encrypt'], encrypt.encrypt)
+    api.commands.register(['decrypt'], encrypt.decrypt)
+    return

onionr/static-data/default-plugins/flow/main.py

@@ -45,9 +45,9 @@ class OnionrFlow:
                 self.flowRunning = False
             if message == "q":
                 self.flowRunning = False
-
+            expireTime = self.myCore._utils.getEpoch() + 43200
             if len(message) > 0:
-                Block(content = message, type = 'txt', core = self.myCore).save()
+                Block(content = message, type = 'txt', expire=expireTime, core = self.myCore).save()
 
         logger.info("Flow is exiting, goodbye")
         return

onionr/static-data/default-plugins/metadataprocessor/info.json

@@ -0,0 +1,5 @@
+{
+    "name" : "metadataprocessor",
+    "version" : "1.0",
+    "author" : "onionr"
+}

onionr/static-data/default-plugins/metadataprocessor/main.py

@@ -0,0 +1,103 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This processes metadata for Onionr blocks
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+# useful libraries
+import logger, config
+import os, sys, json, time, random, shutil, base64, getpass, datetime, re
+from onionrblockapi import Block
+import onionrusers, onionrexceptions
+
+plugin_name = 'metadataprocessor'
+
+# event listeners
+
+def _processUserInfo(api, newBlock):
+    '''
+        Set the username for a particular user, from a signed block by them
+    '''
+    myBlock = newBlock
+    peerName = myBlock.getMetadata('name')
+    try:
+        if len(peerName) > 20:
+            raise onionrexceptions.InvalidMetdata('Peer name specified is too large')
+    except TypeError:
+        pass
+    except onionrexceptions.InvalidMetadata:
+        pass
+    else:
+        api.get_core().setPeerInfo(signer, 'name', peerName)
+        logger.info('%s is now using the name %s.' % (signer, api.get_utils().escapeAnsi(peerName)))
+
+def _processForwardKey(api, myBlock):
+    '''
+        Get the forward secrecy key specified by the user for us to use
+    '''
+    peer = onionrusers.OnionrUser(api.get_core(), myBlock.signer)
+    key = myBlock.getMetadata('newFSKey')
+
+    # We don't need to validate here probably, but it helps
+    if api.get_utils().validatePubKey(key):
+        peer.addForwardKey(key)
+    else:
+        raise onionrexceptions.InvalidPubkey("%s is nota valid pubkey key" % (key,))
+
+def on_processblocks(api):
+    # Generally fired by utils.
+    myBlock = api.data['block']
+    blockType = api.data['type']
+    logger.info('blockType is ' + blockType)
+
+    # Process specific block types
+
+    # userInfo blocks, such as for setting username
+    if blockType == 'userInfo':
+        if api.data['validSig'] == True: # we use == True for type safety
+            _processUserInfo(api, myBlock)
+    # forwardKey blocks, add a new forward secrecy key for a peer
+    elif blockType == 'forwardKey':
+        if api.data['validSig'] == True:
+            _processForwardKey(api, myBlock)
+    # socket blocks
+    elif blockType == 'socket':
+        if api.data['validSig'] == True and myBlock.decrypted: # we check if it is decrypted as a way of seeing if it was for us
+            logger.info('Detected socket advertised to us...')
+            try:
+                address = myBlock.getMetadata('address')
+            except KeyError:
+                raise onionrexceptions.MissingAddress("Missing address for new socket")
+            try:
+                port = myBlock.getMetadata('port')
+            except KeyError:
+                raise ValueError("Missing port for new socket")
+            try:
+                reason = myBlock.getMetadata('reason')
+            except KeyError:
+                raise ValueError("Missing socket reason")
+
+            socketInfo = json.dumps({'peer': api.data['signer'], 'address': address, 'port': port, 'create': False, 'reason': reason})
+            api.get_core().daemonQueueAdd('addSocket', socketInfo)
+        else:
+            logger.warn("socket is not for us or is invalid")
+
+def on_init(api, data = None):
+
+    pluginapi = api
+
+    return

onionr/static-data/default-plugins/pms/main.py

@@ -1,5 +1,5 @@
 '''
-    Onionr - P2P Microblogging Platform & Social network
+    Onionr - P2P Anonymous Storage Network
 
     This default plugin handles private messages in an email like fashion
 '''
@@ -22,9 +22,13 @@
 import logger, config, threading, time, readline, datetime
 from onionrblockapi import Block
 import onionrexceptions, onionrusers
-import locale
+import locale, sys, os
+
 locale.setlocale(locale.LC_ALL, '')
 
+sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
+import sentboxdb # import after path insert
+
 plugin_name = 'pms'
 PLUGIN_VERSION = '0.0.1'
 
@@ -44,22 +48,23 @@ class MailStrings:
         self.mailInstance = mailInstance
 
         self.programTag = 'OnionrMail v%s' % (PLUGIN_VERSION)
-        choices = ['view inbox', 'view sentbox', 'send message', 'help', 'quit']
+        choices = ['view inbox', 'view sentbox', 'send message', 'quit']
         self.mainMenuChoices = choices
         self.mainMenu = '''\n
 -----------------
 1. %s
 2. %s
 3. %s
-4. %s
+4. %s''' % (choices[0], choices[1], choices[2], choices[3])
-5. %s''' % (choices[0], choices[1], choices[2], choices[3], choices[4])
 
 class OnionrMail:
     def __init__(self, pluginapi):
         self.myCore = pluginapi.get_core()
-        #self.dataFolder = pluginapi.get_data_folder()
         self.strings = MailStrings(self)
 
+        self.sentboxTools = sentboxdb.SentBox(self.myCore)
+        self.sentboxList = []
+        self.sentMessages = {}
         return
     
     def inbox(self):
@@ -68,6 +73,7 @@ class OnionrMail:
         pmBlocks = {}
         logger.info('Decrypting messages...')
         choice = ''
+        displayList = []
 
         # this could use a lot of memory if someone has recieved a lot of messages
         for blockHash in self.myCore.getBlocksByType('pm'):
@@ -93,8 +99,10 @@ class OnionrMail:
                     senderDisplay = senderKey
 
                 blockDate = pmBlocks[blockHash].getDate().strftime("%m/%d %H:%M")
-                print('%s. %s - %s: %s' % (blockCount, blockDate, senderDisplay[:12], blockHash))
+                displayList.append('%s. %s - %s: %s' % (blockCount, blockDate, senderDisplay[:12], blockHash))
-
+            #displayList.reverse()
+            for i in displayList:
+                print(i)
             try:
                 choice = logger.readline('Enter a block number, -r to refresh, or -q to stop: ').strip().lower()
             except (EOFError, KeyboardInterrupt):
@@ -121,15 +129,52 @@ class OnionrMail:
                 else:
                     cancel = ''
                     readBlock.verifySig()
-                    print('Message recieved from %s' % (readBlock.signer,))
+                    print('Message recieved from %s' % (self.myCore._utils.bytesToStr(readBlock.signer,)))
                     print('Valid signature:', readBlock.validSig)
                     if not readBlock.validSig:
                         logger.warn('This message has an INVALID signature. ANYONE could have sent this message.')
                         cancel = logger.readline('Press enter to continue to message, or -q to not open the message (recommended).')
                     if cancel != '-q':
                         print(draw_border(self.myCore._utils.escapeAnsi(readBlock.bcontent.decode().strip())))
+                        input("Press enter to continue")
         return
     
+    def sentbox(self):
+        '''
+            Display sent mail messages
+        '''
+        entering = True
+        while entering:
+            self.getSentList()
+            print('Enter block number or -q to return')
+            try:
+                choice = input('>')
+            except (EOFError, KeyboardInterrupt) as e:
+                entering = False
+            else:
+                if choice == '-q':
+                    entering = False
+                else:
+                    try:
+                        self.sentboxList[int(choice) - 1]
+                    except IndexError:
+                        print('Invalid block')
+                    else:
+                        logger.info('Sent to: ' + self.sentMessages[self.sentboxList[int(choice) - 1]][1])
+                        # Print ansi escaped sent message
+                        print(self.myCore._utils.escapeAnsi(self.sentMessages[self.sentboxList[int(choice) - 1]][0]))
+                        input('Press enter to continue...')
+
+        return
+    
+    def getSentList(self):
+        count = 1
+        for i in self.sentboxTools.listSent():
+            self.sentboxList.append(i['hash'])
+            self.sentMessages[i['hash']] = (i['message'], i['peer'])
+            print('%s. %s - %s - %s' % (count, i['hash'], i['peer'][:12], i['date']))
+            count += 1
+
     def draftMessage(self):
         message = ''
         newLine = ''
@@ -166,8 +211,8 @@ class OnionrMail:
 
         print('Inserting encrypted message as Onionr block....')
 
-        self.myCore.insertBlock(message, header='pm', encryptType='asym', asymPeer=recip, sign=True)
+        blockID = self.myCore.insertBlock(message, header='pm', encryptType='asym', asymPeer=recip, sign=True)
-
+        self.sentboxTools.addToSent(blockID, recip, message)
     def menu(self):
         choice = ''
         while True:
@@ -182,12 +227,10 @@ class OnionrMail:
             if choice in (self.strings.mainMenuChoices[0], '1'):
                 self.inbox()
             elif choice in (self.strings.mainMenuChoices[1], '2'):
-                logger.warn('not implemented yet')
+                self.sentbox()
             elif choice in (self.strings.mainMenuChoices[2], '3'):
                 self.draftMessage()
             elif choice in (self.strings.mainMenuChoices[3], '4'):
-                logger.warn('not implemented yet')
-            elif choice in (self.strings.mainMenuChoices[4], '5'):
                 logger.info('Goodbye.')
                 break
             elif choice == '':

onionr/static-data/default-plugins/pms/sentboxdb.py

@@ -0,0 +1,62 @@
+'''
+    Onionr - P2P Microblogging Platform & Social network
+
+    This file handles the sentbox for the mail plugin
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sqlite3, os
+import core
+class SentBox:
+    def __init__(self, mycore):
+        assert isinstance(mycore, core.Core)
+        self.dbLocation = mycore.dataDir + 'sentbox.db'
+        if not os.path.exists(self.dbLocation):
+            self.createDB()
+        self.conn = sqlite3.connect(self.dbLocation)
+        self.cursor = self.conn.cursor()
+        self.core = mycore
+        return
+    
+    def createDB(self):
+        conn = sqlite3.connect(self.dbLocation)
+        cursor = conn.cursor()
+        cursor.execute('''CREATE TABLE sent(
+            hash id not null,
+            peer text not null,
+            message text not null,
+            date int not null
+            );
+        ''')
+        conn.commit()
+        return
+
+    def listSent(self):
+        retData = []
+        for entry in self.cursor.execute('SELECT * FROM sent;'):
+            retData.append({'hash': entry[0], 'peer': entry[1], 'message': entry[2], 'date': entry[3]})
+        return retData
+    
+    def addToSent(self, blockID, peer, message):
+        args = (blockID, peer, message, self.core._utils.getEpoch())
+        self.cursor.execute('INSERT INTO sent VALUES(?, ?, ?, ?)', args)
+        self.conn.commit()
+        return
+    
+    def removeSent(self, blockID):
+        args = (blockID,)
+        self.cursor.execute('DELETE FROM sent where hash=?', args)
+        self.conn.commit()
+        return

onionr/static-data/default_config.json

@@ -2,6 +2,8 @@
     "general" : {
         "dev_mode" : true,
         "display_header" : true,
+        "minimum_block_pow": 5,
+        "minimum_send_pow": 5,
 
         "direct_connect" : {
             "respond" : true,
@@ -32,11 +34,10 @@
     "client" : {
 
     },
-
+    "log": {
-    "log" : {
+        "file": {
-        "file" : {
+            "output": false,
-            "output" : true,
+            "path": "data/output.log"
-            "path" : "data/output.log"
         },
 
         "console" : {

onionr/static-data/index.html

@@ -2,6 +2,6 @@
 
 <p>The content on this server is not necessarily created by the server owner, and was not necessarily stored specifically with the owner's knowledge of its contents.</p>
 
-<p>Onionr is a decentralized, distributed data storage system, that anyone can insert data into.</p>
+<p>Onionr is a decentralized data storage system that anyone can insert data into.</p>
 
 <p>To learn more about Onionr, see the website at <a href="https://onionr.voidnet.tech/">https://Onionr.VoidNet.tech/</a></p>

onionr/tests.py

@@ -14,7 +14,7 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import unittest, sys, os, base64, tarfile, shutil, simplecrypt, logger
+import unittest, sys, os, base64, tarfile, shutil, logger
 
 class OnionrTests(unittest.TestCase):
     def testPython3(self):
@@ -61,36 +61,6 @@ class OnionrTests(unittest.TestCase):
         else:
             self.assertTrue(False)
 
-    def testData_b_Encrypt(self):
-        self.assertTrue(True)
-        return
-
-        logger.debug('-'*26 + '\n')
-        logger.info('Running data dir encrypt test...')
-
-        import core
-        myCore = core.Core()
-        myCore.dataDirEncrypt('password')
-        if os.path.exists('data-encrypted.dat'):
-            self.assertTrue(True)
-        else:
-            self.assertTrue(False)
-
-    def testData_a_Decrypt(self):
-        self.assertTrue(True)
-        return
-
-        logger.debug('-'*26 + '\n')
-        logger.info('Running data dir decrypt test...')
-
-        import core
-        myCore = core.Core()
-        myCore.dataDirDecrypt('password')
-        if os.path.exists('data/'):
-            self.assertTrue(True)
-        else:
-            self.assertTrue(False)
-
     def testConfig(self):
         logger.debug('-'*26 + '\n')
         logger.info('Running simple configuration test...')

requirements.txt

@@ -1,9 +1,9 @@
 urllib3==1.23
-requests==2.18.4
+requests==2.20.0
 PyNaCl==1.2.1
 gevent==1.3.6
 sha3==0.2.1
 defusedxml==0.5.0
-simple_crypt==4.1.7
 Flask==1.0.2
 PySocks==1.6.8
+stem==1.6.0

setprofile.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+ONIONR_HOME=.
+if [ $# -gt 0 ]; then
+  ONIONR_HOME=$1
+export ONIONR_HOME
+echo "set ONIONR_HOME to $ONIONR_HOME"
+fi