Merge pull request #10 from beardog108/communicator-multithreading

Communicator multithreading

.gitignore

@@ -11,3 +11,4 @@ onionr/gnupg/*
 run.sh
 onionr/data-encrypted.dat
 onionr/.onionr-lock
+core

Makefile

@@ -16,6 +16,8 @@ uninstall:
 	sudo rm -f /usr/bin/onionr
 
 test:
+	@./RUN-LINUX.sh stop
+	@sleep 1
 	@rm -rf onionr/data-backup
 	@mv onionr/data onionr/data-backup | true > /dev/null 2>&1
 	-@cd onionr; ./tests.py; ./cryptotests.py;
@@ -30,4 +32,9 @@ soft-reset:
 reset:
 	@echo "Hard-resetting Onionr..."
 	rm -rf onionr/data/ | true > /dev/null 2>&1
+	#@./RUN-LINUX.sh version | grep -v "Failed" --color=always
+
+plugins-reset:
+	@echo "Resetting plugins..."
+	rm -rf onionr/data/plugins/ | true > /dev/null 2>&1
 	@./RUN-LINUX.sh version | grep -v "Failed" --color=always

onionr/api.py

@@ -227,8 +227,8 @@ class API:
                     response = 'none'
                 resp = Response(response)
             elif action == 'kex':
-                peers = self._core.listPeers()
-                response = ','.join(self._core.listPeers())
+                peers = self._core.listPeers(getPow=True)
+                response = ','.join(peers)
                 resp = Response(response)
             else:
                 resp = Response("")

onionr/bitpeer

@@ -1 +0,0 @@
-Subproject commit a74e826e9c69e643ead7950f9f76a05ab8664ddc

onionr/btc.py

@@ -1,44 +0,0 @@
-'''
-    Onionr - P2P Microblogging Platform & Social network
-
-    Handle bitcoin operations
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-from bitpeer.node import *
-from bitpeer.storage.shelve import ShelveStorage
-import logging, time
-import socks, sys
-class OnionrBTC:
-	def __init__(self, lastBlock='00000000000000000021ee6242d08e3797764c9258e54e686bc2afff51baf599', lastHeight=510613, torP=9050):
-		stream = logging.StreamHandler()
-		logger = logging.getLogger('halfnode')
-		logger.addHandler(stream)
-		logger.setLevel (10)
-
-		LASTBLOCK = lastBlock
-		LASTBLOCKINDEX = lastHeight
-		self.node = Node ('BTC', ShelveStorage ('data/btc-blocks.db'), lastblockhash=LASTBLOCK, lastblockheight=LASTBLOCKINDEX, torPort=torP)
-
-		self.node.bootstrap ()
-		self.node.connect ()
-		self.node.loop ()
-
-if __name__ == "__main__":
-    torPort = int(sys.argv[1])
-    bitcoin = OnionrBTC(torPort)
-    while True:
-        print(bitcoin.node.getBlockHash(bitcoin.node.getLastBlockHeight())) # Using print on purpose, do not change to logger
-        time.sleep(5)

onionr/communicator.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
 '''
-Onionr - P2P Microblogging Platform & Social network.
+    Onionr - P2P Microblogging Platform & Social network.
 
-This file contains both the OnionrCommunicate class for communcating with peers
-and code to operate as a daemon, getting commands from the command queue database (see core.Core.daemonQueue)
+    This file contains both the OnionrCommunicate class for communcating with peers
+    and code to operate as a daemon, getting commands from the command queue database (see core.Core.daemonQueue)
 '''
 '''
     This program is free software: you can redistribute it and/or modify
@@ -19,8 +19,8 @@ and code to operate as a daemon, getting commands from the command queue databas
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import sqlite3, requests, hmac, hashlib, time, sys, os, math, logger, urllib.parse, base64, binascii, random, json
-import core, onionrutils, onionrcrypto, netcontroller, onionrproofs, btc, config, onionrplugins as plugins
+import sqlite3, requests, hmac, hashlib, time, sys, os, math, logger, urllib.parse, base64, binascii, random, json, threading
+import core, onionrutils, onionrcrypto, netcontroller, onionrproofs, config, onionrplugins as plugins
 
 class OnionrCommunicate:
     def __init__(self, debug, developmentMode):
@@ -40,28 +40,27 @@ class OnionrCommunicate:
         self.ignoredHashes = []
 
         self.highFailureAmount = 7
-        '''
-        logger.info('Starting Bitcoin Node... with Tor socks port:' + str(sys.argv[2]), timestamp=True)
-        try:
-            self.bitcoin = btc.OnionrBTC(torP=int(sys.argv[2]))
-        except _gdbm.error:
-            pass
-        logger.info('Bitcoin Node started, on block: ' + self.bitcoin.node.getBlockHash(self.bitcoin.node.getLastBlockHeight()), timestamp=True)
-        '''
-        #except:
-        #logger.fatal('Failed to start Bitcoin Node, exiting...')
-        #exit(1)
 
-        blockProcessTimer = 0
-        blockProcessAmount = 5
-        highFailureTimer = 0
-        highFailureRate = 10
-        heartBeatTimer = 0
-        heartBeatRate = 0
-        pexTimer = 25 # How often we should check for new peers
-        pexCount = 0
+        self.communicatorThreads = 0
+        self.maxThreads = 75
+        self.processBlocksThreads = 0
+        self.lookupBlocksThreads = 0
+
+        self.blocksProcessing = [] # list of blocks currently processing, to avoid trying a block twice at once in 2 seperate threads
+        self.peerStatus = {} # network actions (active requests) for peers used mainly to prevent conflicting actions in threads
+
+        self.communicatorTimers = {} # communicator timers, name: rate (in seconds)
+        self.communicatorTimerCounts = {}
+        self.communicatorTimerFuncs = {}
+
+        self.registerTimer('blockProcess', 20)
+        self.registerTimer('highFailure', 10)
+        self.registerTimer('heartBeat', 10)
+        self.registerTimer('pex', 120)
         logger.debug('Communicator debugging enabled.')
-        torID = open('data/hs/hostname').read()
+
+        with open('data/hs/hostname', 'r') as torID:
+            todID = torID.read()
 
         apiRunningCheckRate = 10
         apiRunningCheckCount = 0
@@ -77,24 +76,44 @@ class OnionrCommunicate:
         while True:
             command = self._core.daemonQueue()
             # Process blocks based on a timer
-            blockProcessTimer += 1
-            heartBeatTimer += 1
-            pexCount += 1
-            if highFailureTimer == highFailureRate:
-                highFailureTimer = 0
+            self.timerTick() 
+            # TODO: migrate below if statements to be own functions which are called in the above timerTick() function
+            if self.communicatorTimers['highFailure'] == self.communicatorTimerCounts['highFailure']:
+                self.communicatorTimerCounts['highFailure'] = 0
                 for i in self.peerData:
                     if self.peerData[i]['failCount'] >= self.highFailureAmount:
                         self.peerData[i]['failCount'] -= 1
-            if pexTimer == pexCount:
-                self.getNewPeers()
-                pexCount = 0 # TODO: do not reset timer if low peer count
-            if heartBeatRate == heartBeatTimer:
+            if self.communicatorTimers['pex'] == self.communicatorTimerCounts['pex']:
+                pT1 = threading.Thread(target=self.getNewPeers, name="pT1")
+                pT1.start()
+                pT2 = threading.Thread(target=self.getNewPeers, name="pT2")
+                pT2.start()
+                self.communicatorTimerCounts['pex'] = 0# TODO: do not reset timer if low peer count
+            if self.communicatorTimers['heartBeat'] == self.communicatorTimerCounts['heartBeat']:
                 logger.debug('Communicator heartbeat')
-                heartBeatTimer = 0
-            if blockProcessTimer == blockProcessAmount:
-                self.lookupBlocks()
-                self.processBlocks()
-                blockProcessTimer = 0
+                self.communicatorTimerCounts['heartBeat'] = 0
+            if self.communicatorTimers['blockProcess'] == self.communicatorTimerCounts['blockProcess']:
+                lT1 = threading.Thread(target=self.lookupBlocks, name="lt1", args=(True,))
+                lT2 = threading.Thread(target=self.lookupBlocks, name="lt2", args=(True,))
+                lT3 = threading.Thread(target=self.lookupBlocks, name="lt3", args=(True,))
+                lT4 = threading.Thread(target=self.lookupBlocks, name="lt4", args=(True,))
+                pbT1 = threading.Thread(target=self.processBlocks, name='pbT1', args=(True,))
+                pbT2 = threading.Thread(target=self.processBlocks, name='pbT2', args=(True,))
+                pbT3 = threading.Thread(target=self.processBlocks, name='pbT3', args=(True,))
+                pbT4 = threading.Thread(target=self.processBlocks, name='pbT4', args=(True,))
+                if (self.maxThreads - 8) >= threading.active_count():
+                    lT1.start()
+                    lT2.start()
+                    lT3.start()
+                    lT4.start()
+                    pbT1.start()
+                    pbT2.start()
+                    pbT3.start()
+                    pbT4.start()
+                    self.communicatorTimerCounts['blockProcess'] = 0
+                else:
+                    logger.debug(threading.active_count())
+                    logger.debug('Too many threads.')
             if command != False:
                 if command[0] == 'shutdown':
                     logger.info('Daemon received exit command.', timestamp=True)
@@ -114,6 +133,8 @@ class OnionrCommunicate:
                 elif command[0] == 'runCheck':
                     logger.info('Status check; looks good.')
                     open('data/.runcheck', 'w+').close()
+                elif command[0] == 'kex':
+                    self.pexCount = pexTimer - 1
                 elif command[0] == 'event':
                     # todo
                     pass
@@ -165,6 +186,28 @@ class OnionrCommunicate:
     connection_handlers = {}
     id_peer_cache = {}
 
+    def registerTimer(self, timerName, rate, timerFunc=None):
+        '''Register a communicator timer'''
+        self.communicatorTimers[timerName] = rate
+        self.communicatorTimerCounts[timerName] = 0
+        self.communicatorTimerFuncs[timerName] = timerFunc
+    
+    def timerTick(self):
+        '''Increments timers "ticks" and calls funcs if applicable'''
+        tName = ''
+        for i in self.communicatorTimers.items():
+            tName = i[0]
+            self.communicatorTimerCounts[tName] += 1
+
+            if self.communicatorTimerCounts[tName] == self.communicatorTimers[tName]:
+                try:
+                    self.communicatorTimerFuncs[tName]()
+                except TypeError:
+                    pass
+                else:
+                    self.communicatorTimerCounts[tName] = 0
+
+
     def get_connection_handlers(self, name = None):
         '''
             Returns a list of callback handlers by name, or, if name is None, it returns all handlers.
@@ -174,7 +217,7 @@ class OnionrCommunicate:
             return self.connection_handlers
         elif name in self.connection_handlers:
             return self.connection_handlers[name]
-        else
+        else:
             return list()
 
     def add_connection_handler(self, name, handler):
@@ -274,7 +317,7 @@ class OnionrCommunicate:
 
             events.event('outgoing_direct_connection', data = {'callback' : True, 'communicator' : self, 'data' : data, 'id' : identifier, 'token' : token, 'peer' : peer, 'callback' : callback, 'log' : log})
 
-            logger.debug('Direct connection (identifier: "%s"): %s' + (identifier, data_str))
+            logger.debug('Direct connection (identifier: "%s"): %s' % (identifier, data_str))
             try:
                 self.performGet('directMessage', peer, data_str)
             except:
@@ -360,10 +403,10 @@ class OnionrCommunicate:
 
     def getNewPeers(self):
         '''
-            Get new peers and keys
+            Get new peers and ed25519 keys
         '''
 
-        peersCheck = 5 # Amount of peers to ask for new peers + keys
+        peersCheck = 1 # Amount of peers to ask for new peers + keys
         peersChecked = 0
         peerList = list(self._core.listAdders()) # random ordered list of peers
         newKeys = []
@@ -380,40 +423,49 @@ class OnionrCommunicate:
         while peersCheck > peersChecked:
             #i = secrets.randbelow(maxN) # cant use prior to 3.6
             i = random.randint(0, maxN)
-            logger.info('Using ' + peerList[i] + ' to find new peers', timestamp=True)
+
+            try:
+                if self.peerStatusTaken(peerList[i], 'pex') or self.peerStatusTaken(peerList[i], 'kex'):
+                    continue
+            except IndexError:
+                pass
+
+            logger.info('Using %s to find new peers...' % peerList[i], timestamp=True)
 
             try:
                 newAdders = self.performGet('pex', peerList[i], skipHighFailureAddress=True)
-                logger.debug('Attempting to merge address: ')
-                logger.debug(newAdders)
-                self._utils.mergeAdders(newAdders)
+                if not newAdders is False: # keep the is False thing in there, it might not be bool
+                    logger.debug('Attempting to merge address: %s' % str(newAdders))
+                    self._utils.mergeAdders(newAdders)
             except requests.exceptions.ConnectionError:
-                logger.info(peerList[i] + ' connection failed', timestamp=True)
+                logger.info('%s connection failed' % peerList[i], timestamp=True)
                 continue
             else:
                 try:
-                    logger.info('Using ' + peerList[i] + ' to find new keys')
+                    logger.info('Using %s to find new keys...' % peerList[i])
                     newKeys = self.performGet('kex', peerList[i], skipHighFailureAddress=True)
-                    logger.debug('Attempting to merge pubkey: ')
-                    logger.debug(newKeys)
+                    logger.debug('Attempting to merge pubkey: %s' % str(newKeys))
                     # TODO: Require keys to come with POW token (very large amount of POW)
                     self._utils.mergeKeys(newKeys)
                 except requests.exceptions.ConnectionError:
-                    logger.info(peerList[i] + ' connection failed', timestamp=True)
+                    logger.info('%s connection failed' % peerList[i], timestamp=True)
                     continue
                 else:
                     peersChecked += 1
         return
 
-    def lookupBlocks(self):
+    def lookupBlocks(self, isThread=False):
         '''
             Lookup blocks and merge new ones
         '''
-
+        if isThread:
+            self.lookupBlocksThreads += 1
         peerList = self._core.listAdders()
-        blocks = ''
+        blockList = list()
 
         for i in peerList:
+            if self.peerStatusTaken(i, 'getBlockHashes') or self.peerStatusTaken(i, 'getDBHash'):
+                continue
             try:
                 if self.peerData[i]['failCount'] >= self.highFailureAmount:
                     continue
@@ -423,66 +475,70 @@ class OnionrCommunicate:
             lastDB = self._core.getAddressInfo(i, 'DBHash')
 
             if lastDB == None:
-                logger.debug('Fetching hash from ' + str(i) + ', no previous known.')
+                logger.debug('Fetching hash from %s, no previous known.' % str(i))
             else:
-                logger.debug('Fetching hash from ' + str(i) + ', ' + str(lastDB) + ' last known')
+                logger.debug('Fetching hash from %s, %s last known' % (str(i), str(lastDB)))
 
             currentDB = self.performGet('getDBHash', i)
 
             if currentDB != False:
-                logger.debug(i + " hash db (from request): " + currentDB)
+                logger.debug('%s hash db (from request): %s' % (str(i), str(currentDB)))
             else:
-                logger.warn("Error getting hash db status for " + i)
+                logger.warn('Failed to get hash db status for %s' % str(i))
 
             if currentDB != False:
                 if lastDB != currentDB:
-                    logger.debug('Fetching hash from ' + i + ' - ' + currentDB + ' current hash.')
+                    logger.debug('Fetching hash from %s - %s current hash.' % (str(i), currentDB))
                     try:
-                        blocks += self.performGet('getBlockHashes', i)
+                        blockList.append(self.performGet('getBlockHashes', i))
                     except TypeError:
-                        logger.warn('Failed to get data hash from ' + i)
+                        logger.warn('Failed to get data hash from %s' % str(i))
                         self.peerData[i]['failCount'] -= 1
                 if self._utils.validateHash(currentDB):
                     self._core.setAddressInfo(i, "DBHash", currentDB)
 
-        if len(blocks.strip()) != 0:
+        if len(blockList) != 0:
             pass
-            #logger.debug('BLOCKS:' + blocks)
-
-        blockList = blocks.split('\n')
 
         for i in blockList:
             if len(i.strip()) == 0:
                 continue
-            if self._utils.hasBlock(i):
-                continue
+            try:
+                if self._utils.hasBlock(i):
+                    continue
+            except:
+                logger.warn('Invalid hash') # TODO: move below validate hash check below
+                pass
             if i in self.ignoredHashes:
                 continue
 
             #logger.debug('Exchanged block (blockList): ' + i)
             if not self._utils.validateHash(i):
                 # skip hash if it isn't valid
-                logger.warn('Hash ' + i + ' is not valid')
+                logger.warn('Hash %s is not valid' % str(i))
                 continue
             else:
                 self.newHashes[i] = 0
-                logger.debug('Adding ' +  i + ' to hash database...')
+                logger.debug('Adding %s to hash database...' % str(i))
                 self._core.addToBlockDB(i)
-
+        self.lookupBlocksThreads -= 1
         return
 
-    def processBlocks(self):
+    def processBlocks(self, isThread=False):
         '''
             Work with the block database and download any missing blocks
 
             This is meant to be called from the communicator daemon on its timer.
         '''
-
-        for i in self._core.getBlockList(unsaved=True).split("\n"):
+        if isThread:
+            self.processBlocksThreads += 1
+        for i in self._core.getBlockList(unsaved = True):
             if i != "":
-                if i in self.ignoredHashes:
+                if i in self.blocksProcessing or i in self.ignoredHashes:
+                    #logger.debug('already processing ' + i)
                     continue
-
+                else:
+                    self.blocksProcessing.append(i)
                 try:
                     self.newHashes[i]
                 except KeyError:
@@ -490,61 +546,78 @@ class OnionrCommunicate:
 
                 # check if a new hash has been around too long, delete it from database and add it to ignore list
                 if self.newHashes[i] >= self.keepNewHash:
-                    logger.warn('Ignoring block ' + i + ' because it took to long to get valid data.')
+                    logger.warn('Ignoring block %s because it took to long to get valid data.' % str(i))
                     del self.newHashes[i]
                     self._core.removeBlock(i)
                     self.ignoredHashes.append(i)
                     continue
 
                 self.newHashes[i] += 1
-                logger.warn('UNSAVED BLOCK: ' + i)
+                logger.warn('Block is unsaved: %s' % str(i))
                 data = self.downloadBlock(i)
 
-                # if block was successfull gotten (hash already verified)
+                # if block was successfully gotten (hash already verified)
                 if data:
                     del self.newHashes[i] # remove from probation list
 
                     # deal with block metadata
                     blockContent = self._core.getData(i)
+                    try:
+                        blockContent = blockContent.encode()
+                    except AttributeError:
+                        pass
                     try:
                         #blockMetadata = json.loads(self._core.getData(i)).split('}')[0] + '}'
-                        blockMetadata = self._core.getData(i).split(b'}')[0]
+                        blockMetadata = json.loads(blockContent[:blockContent.find(b'\n')].decode())
                         try:
-                            blockMetadata = blockMetadata.decode()
+                            blockMeta2 = json.loads(blockMetadata['meta'])
+                        except KeyError:
+                            blockMeta2 = {'type': ''}
+                            pass
+                        blockContent = blockContent[blockContent.find(b'\n') + 1:]
+                        try:
+                            blockContent = blockContent.decode()
                         except AttributeError:
                             pass
 
-                        blockMetadata = json.loads(blockMetadata + '}')
-
-                        try:
-                            blockMetadata['sig']
-                            blockMetadata['id']
-                        except KeyError:
-                            pass
-
+                        if not self._crypto.verifyPow(blockContent, blockMeta2):
+                            logger.warn("%s has invalid or insufficient proof of work token, deleting..." % str(i))
+                            self._core.removeBlock(i)
+                            continue
                         else:
-                            creator = self._utils.getPeerByHashId(blockMetadata['id'])
-                            try:
-                                creator = creator.decode()
-                            except AttributeError:
-                                pass
+                            if (('sig' in blockMetadata) and ('id' in blockMeta2)): # id doesn't exist in blockMeta2, so this won't workin the first place
 
-                            if self._core._crypto.edVerify(blockContent.split(b'}')[1], creator, blockMetadata['sig'], encodedData=True):
-                                self._core.updateBlockInfo(i, 'sig', 'true')
-                            else:
-                                self._core.updateBlockInfo(i, 'sig', 'false')
+                                #blockData = json.dumps(blockMetadata['meta']) + blockMetadata[blockMetadata.rfind(b'}') + 1:]
+
+                                creator = self._utils.getPeerByHashId(blockMeta2['id'])
+                                try:
+                                    creator = creator.decode()
+                                except AttributeError:
+                                    pass
+
+                                if self._core._crypto.edVerify(blockMetadata['meta'] + blockContent, creator, blockMetadata['sig'], encodedData=True):
+                                    logger.info('%s was signed' % str(i))
+                                    self._core.updateBlockInfo(i, 'sig', 'true')
+                                else:
+                                    logger.warn('%s has an invalid signature' % str(i))
+                                    self._core.updateBlockInfo(i, 'sig', 'false')
                         try:
-                            logger.info('Block type is ' + blockMetadata['type'])
-                            self._core.updateBlockInfo(i, 'dataType', blockMetadata['type'])
+                            logger.info('Block type is %s' % str(blockMeta2['type']))
+                            self._core.updateBlockInfo(i, 'dataType', blockMeta2['type'])
+                            self.removeBlockFromProcessingList(i)
+                            self.removeBlockFromProcessingList(i)
                         except KeyError:
                             logger.warn('Block has no type')
                             pass
                     except json.decoder.JSONDecodeError:
                         logger.warn('Could not decode block metadata')
-                        pass
-
+                        self.removeBlockFromProcessingList(i)
+        self.processBlocksThreads -= 1
         return
 
+    def removeBlockFromProcessingList(self, block):
+        return block in blocksProcessing
+
     def downloadBlock(self, hash, peerTries=3):
         '''
             Download a block from random order of peers
@@ -556,8 +629,11 @@ class OnionrCommunicate:
         peerTryCount = 0
 
         for i in peerList:
-            if self.peerData[i]['failCount'] >= self.highFailureAmount:
-                continue
+            try:
+                if self.peerData[i]['failCount'] >= self.highFailureAmount:
+                    continue
+            except KeyError:
+                pass
             if peerTryCount >= peerTries:
                 break
 
@@ -581,17 +657,11 @@ class OnionrCommunicate:
 
             if digest == hash.strip():
                 self._core.setData(data)
-                logger.info('Successfully obtained data for ' + hash, timestamp=True)
+                logger.info('Successfully obtained data for %s' % str(hash), timestamp=True)
                 retVal = True
                 break
-                '''
-                if data.startswith(b'-txt-'):
-                    self._core.setBlockType(hash, 'txt')
-                    if len(data) < 120:
-                        logger.debug('Block text:\n' + data.decode())
-                '''
             else:
-                logger.warn("Failed to validate " + hash + " " + " hash calculated was " + digest)
+                logger.warn("Failed to validate %s -- hash calculated was %s" % (hash, digest))
                 peerTryCount += 1
 
         return retVal
@@ -614,12 +684,12 @@ class OnionrCommunicate:
             raise Exception("Could not perform self address check in performGet due to not knowing our address")
         if selfCheck:
             if peer.replace('/', '') == self._core.hsAdder:
-                logger.warn('Tried to performget to own hidden service, but selfCheck was not set to false')
+                logger.warn('Tried to performGet to own hidden service, but selfCheck was not set to false')
                 return
 
         # Store peer in peerData dictionary (non permanent)
         if not peer in self.peerData:
-            self.peerData[peer] = {'connectCount': 0, 'failCount': 0, 'lastConnectTime': math.floor(time.time())}
+            self.peerData[peer] = {'connectCount': 0, 'failCount': 0, 'lastConnectTime': self._utils.getEpoch()}
         socksPort = sys.argv[2]
         '''We use socks5h to use tor as DNS'''
         proxies = {'http': 'socks5://127.0.0.1:' + str(socksPort), 'https': 'socks5://127.0.0.1:' + str(socksPort)}
@@ -630,13 +700,14 @@ class OnionrCommunicate:
         try:
             if skipHighFailureAddress and self.peerData[peer]['failCount'] > self.highFailureAmount:
                 retData = False
-                logger.debug('Skipping ' + peer + ' because of high failure rate')
+                logger.debug('Skipping %s because of high failure rate.' % peer)
             else:
-                logger.debug('Contacting ' + peer + ' on port ' + socksPort)
+                self.peerStatus[peer] = action
+                logger.debug('Contacting %s on port %s' % (peer, str(socksPort)))
                 r = requests.get(url, headers=headers, proxies=proxies, timeout=(15, 30))
                 retData = r.text
         except requests.exceptions.RequestException as e:
-            logger.warn(action + " failed with peer " + peer + ": " + str(e))
+            logger.debug("%s failed with peer %s" % (action, peer))
             retData = False
 
         if not retData:
@@ -644,9 +715,20 @@ class OnionrCommunicate:
         else:
             self.peerData[peer]['connectCount'] += 1
             self.peerData[peer]['failCount'] -= 1
-            self.peerData[peer]['lastConnectTime'] = math.floor(time.time())
+            self.peerData[peer]['lastConnectTime'] = self._utils.getEpoch()
+            self._core.setAddressInfo(peer, 'lastConnect', self._utils.getEpoch())
         return retData
 
+    def peerStatusTaken(self, peer, status):
+        '''
+            Returns if we are currently performing a specific action with a peer.
+        '''
+        try:
+            if self.peerStatus[peer] == status:
+                return True
+        except KeyError:
+            pass
+        return False
 
 shouldRun = False
 debug = True

onionr/core.py

@@ -17,11 +17,11 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import sqlite3, os, sys, time, math, base64, tarfile, getpass, simplecrypt, hashlib, nacl, logger, json, netcontroller
+import sqlite3, os, sys, time, math, base64, tarfile, getpass, simplecrypt, hashlib, nacl, logger, json, netcontroller, math
 #from Crypto.Cipher import AES
 #from Crypto import Random
 
-import onionrutils, onionrcrypto, btc, onionrevents as events
+import onionrutils, onionrcrypto, onionrproofs, onionrevents as events
 
 if sys.version_info < (3, 6):
     try:
@@ -73,22 +73,24 @@ class Core:
         except Exception as error:
             logger.error('Failed to initialize core Onionr library.', error=error)
             logger.fatal('Cannot recover from error.')
-            exit(1)
+            sys.exit(1)
         return
 
-    def addPeer(self, peerID, name=''):
+    def addPeer(self, peerID, powID, name=''):
         '''
             Adds a public key to the key database (misleading function name)
-
-            DOES NO SAFETY CHECKS if the ID is valid, but prepares the insertion
         '''
         # This function simply adds a peer to the DB
         if not self._utils.validatePubKey(peerID):
             return False
+        if sys.getsizeof(powID) > 60:
+            logger.warn("POW token for pubkey base64 representation exceeded 60 bytes")
+            return False
+
         conn = sqlite3.connect(self.peerDB)
         hashID = self._crypto.pubKeyHashID(peerID)
         c = conn.cursor()
-        t = (peerID, name, 'unknown', hashID)
+        t = (peerID, name, 'unknown', hashID, powID)
 
         for i in c.execute("SELECT * FROM PEERS where id = '" + peerID + "';"):
             try:
@@ -99,7 +101,7 @@ class Core:
                 pass
             except IndexError:
                 pass
-        c.execute('INSERT INTO peers (id, name, dateSeen, hashID) VALUES(?, ?, ?, ?);', t)
+        c.execute('INSERT INTO peers (id, name, dateSeen, pow, hashID) VALUES(?, ?, ?, ?, ?);', t)
         conn.commit()
         conn.close()
 
@@ -189,7 +191,8 @@ class Core:
             speed int,
             success int,
             DBHash text,
-            failure int
+            failure int,
+            lastConnect int
             );
         ''')
         conn.commit()
@@ -212,7 +215,8 @@ class Core:
             bytesStored int,
             trust int,
             pubkeyExchanged int,
-            hashID);
+            hashID text,
+            pow text not null);
         ''')
         conn.commit()
         conn.close()
@@ -251,7 +255,7 @@ class Core:
 
         return
 
-    def addToBlockDB(self, newHash, selfInsert=False):
+    def addToBlockDB(self, newHash, selfInsert=False, dataSaved=False):
         '''
             Add a hash value to the block db
 
@@ -263,8 +267,8 @@ class Core:
             return
         conn = sqlite3.connect(self.blockDB)
         c = conn.cursor()
-        currentTime = math.floor(time.time())
-        if selfInsert:
+        currentTime = self._utils.getEpoch()
+        if selfInsert or dataSaved:
             selfInsert = 1
         else:
             selfInsert = 0
@@ -275,11 +279,12 @@ class Core:
 
         return
 
-    def getData(self,hash):
+    def getData(self, hash):
         '''
             Simply return the data associated to a hash
         '''
         try:
+            # logger.debug('Opening %s' % (str(self.blockDataLocation) + str(hash) + '.dat'))
             dataFile = open(self.blockDataLocation + hash + '.dat', 'rb')
             data = dataFile.read()
             dataFile.close()
@@ -387,7 +392,7 @@ class Core:
             Add a command to the daemon queue, used by the communication daemon (communicator.py)
         '''
         # Intended to be used by the web server
-        date = math.floor(time.time())
+        date = self._utils.getEpoch()
         conn = sqlite3.connect(self.queueDB)
         c = conn.cursor()
         t = (command, data, date)
@@ -431,7 +436,7 @@ class Core:
         conn.close()
         return addressList
 
-    def listPeers(self, randomOrder=True):
+    def listPeers(self, randomOrder=True, getPow=False):
         '''
             Return a list of public keys (misleading function name)
 
@@ -448,10 +453,19 @@ class Core:
         for i in c.execute(payload):
             try:
                 if len(i[0]) != 0:
-                    peerList.append(i[0])
+                    if getPow:
+                        peerList.append(i[0] + '-' + i[1])
+                    else:
+                        peerList.append(i[0])
             except TypeError:
                 pass
-        peerList.append(self._crypto.pubKey)
+        if getPow:
+            try:
+                peerList.append(self._crypto.pubKey + '-' + self._crypto.pubKeyPowToken)
+            except TypeError:
+                pass
+        else:
+            peerList.append(self._crypto.pubKey)
         conn.close()
         return peerList
 
@@ -513,11 +527,12 @@ class Core:
             success int, 4
             DBHash text, 5
             failure int 6
+            lastConnect 7
         '''
         conn = sqlite3.connect(self.addressDB)
         c = conn.cursor()
         command = (address,)
-        infoNumbers = {'address': 0, 'type': 1, 'knownPeer': 2, 'speed': 3, 'success': 4, 'DBHash': 5, 'failure': 6}
+        infoNumbers = {'address': 0, 'type': 1, 'knownPeer': 2, 'speed': 3, 'success': 4, 'DBHash': 5, 'failure': 6, 'lastConnect': 7}
         info = infoNumbers[info]
         iterCount = 0
         retVal = ''
@@ -539,7 +554,7 @@ class Core:
         c = conn.cursor()
         command = (data, address)
         # TODO: validate key on whitelist
-        if key not in ('address', 'type', 'knownPeer', 'speed', 'success', 'DBHash', 'failure'):
+        if key not in ('address', 'type', 'knownPeer', 'speed', 'success', 'DBHash', 'failure', 'lastConnect'):
             raise Exception("Got invalid database key when setting address info")
         c.execute('UPDATE adders SET ' + key + ' = ? WHERE address=?', command)
         conn.commit()
@@ -565,22 +580,36 @@ class Core:
 
         return
 
-    def getBlockList(self, unsaved = False):
+    def getBlockList(self, unsaved = False): # TODO: Use unsaved
         '''
             Get list of our blocks
         '''
         conn = sqlite3.connect(self.blockDB)
         c = conn.cursor()
-        retData = ''
         if unsaved:
             execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();'
         else:
             execute = 'SELECT hash FROM hashes ORDER BY RANDOM();'
+        rows = list()
         for row in c.execute(execute):
             for i in row:
-                retData += i + "\n"
+                rows.append(i)
 
-        return retData
+        return rows
+
+    def getBlockDate(self, blockHash):
+        '''
+            Returns the date a block was received
+        '''
+        conn = sqlite3.connect(self.blockDB)
+        c = conn.cursor()
+        execute = 'SELECT dateReceived FROM hashes WHERE hash=?;'
+        args = (blockHash,)
+        for row in c.execute(execute, args):
+            for i in row:
+                return int(i)
+
+        return None
 
     def getBlocksByType(self, blockType):
         '''
@@ -588,14 +617,14 @@ class Core:
         '''
         conn = sqlite3.connect(self.blockDB)
         c = conn.cursor()
-        retData = ''
         execute = 'SELECT hash FROM hashes WHERE dataType=?;'
         args = (blockType,)
+        rows = list()
         for row in c.execute(execute, args):
             for i in row:
-                retData += i + "\n"
+                rows.append(i)
 
-        return retData.split('\n')
+        return rows
 
     def setBlockType(self, hash, blockType):
         '''
@@ -630,32 +659,57 @@ class Core:
             Inserts a block into the network
         '''
 
+        powProof = onionrproofs.POW(data)
+        powToken = ''
+        # wait for proof to complete
+        try:
+            while True:
+                powToken = powProof.getResult()
+                if powToken == False:
+                    time.sleep(0.3)
+                    continue
+                powHash = powToken[0]
+                powToken = base64.b64encode(powToken[1])
+                try:
+                    powToken = powToken.decode()
+                except AttributeError:
+                    pass
+                finally:
+                    break
+        except KeyboardInterrupt:
+            logger.warn("Got keyboard interrupt while working on inserting block, stopping.")
+            powProof.shutdown()
+            return ''
+
         try:
             data.decode()
         except AttributeError:
             data = data.encode()
 
         retData = ''
-        metadata = {'type': header}
+        metadata = {'type': header, 'powHash': powHash, 'powToken': powToken}
+        sig = {}
+
+        metadata = json.dumps(metadata)
+        metadata = metadata.encode()
+        signature = ''
 
         if sign:
-            signature = self._crypto.edSign(data, self._crypto.privKey, encodeResult=True)
+            signature = self._crypto.edSign(metadata + b'\n' + data, self._crypto.privKey, encodeResult=True)
             ourID = self._crypto.pubKeyHashID()
             # Convert from bytes on some py versions?
             try:
                 ourID = ourID.decode()
             except AttributeError:
                 pass
-            metadata['id'] = ourID
-            metadata['sig'] = signature
-
+        metadata = {'sig': signature, 'meta': metadata.decode()}
         metadata = json.dumps(metadata)
         metadata = metadata.encode()
 
         if len(data) == 0:
             logger.error('Will not insert empty block')
         else:
-            addedHash = self.setData(metadata + data)
+            addedHash = self.setData(metadata + b'\n' + data)
             self.addToBlockDB(addedHash, selfInsert=True)
             self.setBlockType(addedHash, header)
             retData = addedHash

onionr/default-plugins/pluginmanager/main.py

@@ -1,141 +0,0 @@
-'''
-    This is the future Onionr plugin manager. TODO: Add better description.
-'''
-
-# useful libraries
-import logger, config
-import os, sys, json
-
-plugin_name = 'pluginmanager'
-
-keys_data = {'keys' : {}}
-
-# key functions
-
-def writeKeys():
-    '''
-        Serializes and writes the keystore in memory to file
-    '''
-
-    file = open(keys_file, 'w')
-    file.write(json.dumps(keys_data, indent=4, sort_keys=True))
-    file.close()
-
-def readKeys():
-    '''
-        Loads the keystore into memory
-    '''
-
-    global keys_data
-    keys_data = json.loads(open(keys_file).read())
-    return keys_data
-
-def getKey(plugin):
-    '''
-        Returns the public key for a given plugin
-    '''
-
-    readKeys()
-    return (keys_data['keys'][plugin] if plugin in keys_data['keys'] else None)
-
-def saveKey(plugin, key):
-    '''
-        Saves the public key for a plugin to keystore
-    '''
-
-    keys_data['keys'][plugin] = key
-    writeKeys()
-
-def check():
-    '''
-        Checks to make sure the keystore file still exists
-    '''
-
-    global keys_file
-    keys_file = pluginapi.plugins.get_data_folder(plugin_name) + 'keystore.json'
-    if not os.path.isfile(keys_file):
-        writeKeys()
-
-# command handlers
-
-def help():
-    logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin> [public key/block hash]')
-
-def commandInstallPlugin():
-    logger.warn('This feature is not functional or is still in development.')
-    if len(sys.argv) >= 3:
-        check()
-
-        pluginname = sys.argv[2]
-        pkobh = None # public key or block hash
-
-        if len(sys.argv) >= 4:
-            # public key or block hash specified
-            pkobh = sys.argv[3]
-        else:
-            # none specified, check if in config file
-            pkobh = getKey(pluginname)
-
-        if pkobh is None:
-            logger.error('No key for this plugin found in keystore, please specify.')
-            help()
-            return True
-
-        valid_hash = pluginapi.get_utils().validateHash(pkobh)
-        real_block = False
-        valid_key = pluginapi.get_utils().validatePubKey(pkobh)
-        real_key = False
-
-        if valid_hash:
-            real_block = pluginapi.get_utils().hasBlock(pkobh)
-        elif valid_key:
-            real_key = pluginapi.get_utils().hasKey(pkobh)
-
-        blockhash = None
-
-        if valid_hash and not real_block:
-            logger.error('Block hash not found. Perhaps it has not been synced yet?')
-            logger.debug('Is valid hash, but does not belong to a known block.')
-            return True
-        elif valid_hash and real_block:
-            blockhash = str(pkobh)
-            logger.debug('Using block %s...' % blockhash)
-        elif valid_key and not real_key:
-            logger.error('Public key not found. Try adding the node by address manually, if possible.')
-            logger.debug('Is valid key, but the key is not a known one.')
-        elif valid_key and real_key:
-            publickey = str(pkobh)
-            logger.debug('Using public key %s...' % publickey)
-
-            saveKey(pluginname, pkobh)
-        else:
-            logger.error('Unknown data "%s"; must be public key or block hash.' % str(pkobh))
-            return
-    else:
-        help()
-
-    return True
-
-def commandUninstallPlugin():
-    logger.info('This feature has not been created yet. Please check back later.')
-    return
-
-def commandSearchPlugin():
-    logger.info('This feature has not been created yet. Please check back later.')
-    return
-
-# event listeners
-
-def on_init(api, data = None):
-    global pluginapi
-    pluginapi = api
-    check()
-
-    # register some commands
-    api.commands.register(['install-plugin', 'installplugin', 'plugin-install', 'install', 'plugininstall'], commandInstallPlugin)
-    api.commands.register(['remove-plugin', 'removeplugin', 'plugin-remove', 'uninstall-plugin', 'uninstallplugin', 'plugin-uninstall', 'uninstall', 'remove', 'pluginremove'], commandUninstallPlugin)
-    api.commands.register(['search', 'filter-plugins', 'search-plugins', 'searchplugins', 'search-plugin', 'searchplugin', 'findplugin', 'find-plugin', 'filterplugin', 'plugin-search', 'pluginsearch'], commandSearchPlugin)
-
-    # add help menus once the features are actually implemented
-
-    return

onionr/logger.py

@@ -222,7 +222,7 @@ def error(data, error=None, timestamp=True):
     if not error is None:
         debug('Error: ' + str(error) + parse_error())
 
-# fatal: when the something so bad has happened that the prorgam must stop
+# fatal: when the something so bad has happened that the program must stop
 def fatal(data, timestamp=True):
     if get_level() <= LEVEL_FATAL:
         log('#', data, colors.bg.red + colors.fg.green + colors.bold, timestamp=timestamp)

onionr/netcontroller.py

@@ -89,13 +89,17 @@ DataDirectory data/tordata/
             torVersion.kill()
 
         # wait for tor to get to 100% bootstrap
-        for line in iter(tor.stdout.readline, b''):
-            if 'Bootstrapped 100%: Done' in line.decode():
-                break
-            elif 'Opening Socks listener' in line.decode():
-                logger.debug(line.decode().replace('\n', ''))
-        else:
-            logger.fatal('Failed to start Tor. Try killing any other Tor processes owned by this user.')
+        try:
+            for line in iter(tor.stdout.readline, b''):
+                if 'Bootstrapped 100%: Done' in line.decode():
+                    break
+                elif 'Opening Socks listener' in line.decode():
+                    logger.debug(line.decode().replace('\n', ''))
+            else:
+                logger.fatal('Failed to start Tor. Try killing any other Tor processes owned by this user.')
+                return False
+        except KeyboardInterrupt:
+            logger.fatal("Got keyboard interrupt")
             return False
 
         logger.info('Finished starting Tor', timestamp=True)

onionr/onionr.py

@@ -25,9 +25,10 @@ import sys
 if sys.version_info[0] == 2 or sys.version_info[1] < 5:
     print('Error, Onionr requires Python 3.4+')
     sys.exit(1)
-import os, base64, random, getpass, shutil, subprocess, requests, time, platform, datetime, re
+import os, base64, random, getpass, shutil, subprocess, requests, time, platform, datetime, re, json, getpass
 from threading import Thread
 import api, core, config, logger, onionrplugins as plugins, onionrevents as events
+import onionrutils
 from onionrutils import OnionrUtils
 from netcontroller import NetController
 
@@ -64,7 +65,7 @@ class Onionr:
         else:
             # the default config file doesn't exist, try hardcoded config
             config.set_config({'devmode': True, 'log': {'file': {'output': True, 'path': 'data/output.log'}, 'console': {'output': True, 'color': True}}})
-        if not exists:
+        if not data_exists:
             config.save()
         config.reload() # this will read the configuration file into memory
 
@@ -107,12 +108,11 @@ class Onionr:
                 if not os.path.exists('data/blocks/'):
                     os.mkdir('data/blocks/')
 
-                # Copy default plugins into plugins folder
-
+            # Copy default plugins into plugins folder
             if not os.path.exists(plugins.get_plugins_folder()):
-                if os.path.exists('default-plugins/'):
-                    names = [f for f in os.listdir("default-plugins/") if not os.path.isfile(f)]
-                    shutil.copytree('default-plugins/', plugins.get_plugins_folder())
+                if os.path.exists('static-data/default-plugins/'):
+                    names = [f for f in os.listdir("static-data/default-plugins/") if not os.path.isfile(f)]
+                    shutil.copytree('static-data/default-plugins/', plugins.get_plugins_folder())
 
                     # Enable plugins
                     for name in names:
@@ -134,7 +134,7 @@ class Onionr:
 
         # Get configuration
 
-        if not exists:
+        if not data_exists:
             # Generate default config
             # Hostname should only be set if different from 127.x.x.x. Important for DNS rebinding attack prevention.
             if self.debug:
@@ -153,6 +153,8 @@ class Onionr:
             'config': self.configure,
             'start': self.start,
             'stop': self.killDaemon,
+            'status': self.showStats,
+            'statistics': self.showStats,
             'stats': self.showStats,
 
             'enable-plugin': self.enablePlugin,
@@ -191,6 +193,8 @@ class Onionr:
             'addaddress': self.addAddress,
             'addfile': self.addFile,
 
+            'importblocks': self.onionrUtils.importNewBlocks,
+
             'introduce': self.onionrCore.introduceNode,
             'connect': self.addAddress
         }
@@ -212,11 +216,12 @@ class Onionr:
             'pm': 'Adds a private message to block',
             'get-pms': 'Shows private messages sent to you',
             'addfile': 'Create an Onionr block from a file',
+            'importblocks': 'import blocks from the disk (Onionr is transport-agnostic!)',
             'introduce': 'Introduce your node to the public Onionr network',
         }
 
         # initialize plugins
-        events.event('init', onionr = self)
+        events.event('init', onionr = self, threaded = False)
 
         command = ''
         try:
@@ -389,7 +394,8 @@ class Onionr:
         addedHash = self.onionrCore.insertBlock(messageToAdd, header='txt')
         #self.onionrCore.addToBlockDB(addedHash, selfInsert=True)
         #self.onionrCore.setBlockType(addedHash, 'txt')
-        logger.info("Message inserted as as block %s" % addedHash)
+        if addedHash != '':
+            logger.info("Message inserted as as block %s" % addedHash)
         return
 
     def getPMs(self):
@@ -457,7 +463,10 @@ class Onionr:
 
                     os.makedirs(plugins.get_plugins_folder(plugin_name))
                     with open(plugins.get_plugins_folder(plugin_name) + '/main.py', 'a') as main:
-                        main.write(open('static-data/default_plugin.py').read().replace('$user', os.getlogin()).replace('$date', datetime.datetime.now().strftime('%Y-%m-%d')))
+                        main.write(open('static-data/default_plugin.py').read().replace('$user', os.getlogin()).replace('$date', datetime.datetime.now().strftime('%Y-%m-%d')).replace('$name', plugin_name))
+
+                    with open(plugins.get_plugins_folder(plugin_name) + '/info.json', 'a') as main:
+                        main.write(json.dumps({'author' : 'anonymous', 'description' : 'the default description of the plugin', 'version' : '1.0'}))
 
                     logger.info('Enabling plugin "%s"...' % plugin_name)
                     plugins.enable(plugin_name, self)
@@ -550,8 +559,54 @@ class Onionr:
             Displays statistics and exits
         '''
 
-        logger.info('Our pubkey: ' + self.onionrCore._crypto.pubKey)
-        logger.info('Our address: ' + self.get_hostname())
+        try:
+            # define stats messages here
+            messages = {
+                # info about local client
+                'Onionr Daemon Status' : ((logger.colors.fg.green + 'Online') if self.onionrUtils.isCommunicatorRunning(timeout = 2) else logger.colors.fg.red + 'Offline'),
+                'Public Key' : self.onionrCore._crypto.pubKey,
+                'Address' : self.get_hostname(),
+
+                # file and folder size stats
+                'div1' : True, # this creates a solid line across the screen, a div
+                'Total Block Size' : onionrutils.humanSize(onionrutils.size('data/blocks/')),
+                'Total Plugin Size' : onionrutils.humanSize(onionrutils.size('data/plugins/')),
+                'Log File Size' : onionrutils.humanSize(onionrutils.size('data/output.log')),
+
+                # count stats
+                'div2' : True,
+                'Known Peers Count' : str(len(self.onionrCore.listPeers())),
+                'Enabled Plugins Count' : str(len(config.get('plugins')['enabled'])) + ' / ' + str(len(os.listdir('data/plugins/')))
+            }
+
+            # color configuration
+            colors = {
+                'title' : logger.colors.bold,
+                'key' : logger.colors.fg.lightgreen,
+                'val' : logger.colors.fg.green,
+                'border' : logger.colors.fg.lightblue,
+
+                'reset' : logger.colors.reset
+            }
+
+            # pre-processing
+            maxlength = 0
+            for key, val in messages.items():
+                if not (type(val) is bool and val is True):
+                    maxlength = max(len(key), maxlength)
+
+            # generate stats table
+            logger.info(colors['title'] + 'Onionr v%s Statistics' % ONIONR_VERSION + colors['reset'])
+            logger.info(colors['border'] + '─' * (maxlength + 1) + '┐' + colors['reset'])
+            for key, val in messages.items():
+                if not (type(val) is bool and val is True):
+                    logger.info(colors['key'] + str(key).rjust(maxlength) + colors['reset'] + colors['border'] + ' │ ' + colors['reset'] + colors['val'] + str(val) + colors['reset'])
+                else:
+                    logger.info(colors['border'] + '─' * (maxlength + 1) + '┤' + colors['reset'])
+            logger.info(colors['border'] + '─' * (maxlength + 1) + '┘' + colors['reset'])
+        except Exception as e:
+            logger.error('Failed to generate statistics table.', error = e, timestamp = False)
+
         return
 
     def showHelp(self, command = None):

onionr/onionrblockapi.py

@@ -0,0 +1,443 @@
+'''
+    Onionr - P2P Microblogging Platform & Social network.
+
+    This class contains the OnionrBlocks class which is a class for working with Onionr blocks
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+import core as onionrcore, logger
+import json, os, datetime
+
+class Block:
+    def __init__(self, hash = None, core = None):
+        '''
+            Initializes Onionr
+
+            Inputs:
+            - hash (str): the hash of the block to be imported, if any
+            - core (Core/str):
+              - if (Core): this is the Core instance to be used, don't create a new one
+              - if (str): treat `core` as the block content, and instead, treat `hash` as the block type
+
+            Outputs:
+            - (Block): the new Block instance
+        '''
+
+        # input from arguments
+        if (type(hash) == str) and (type(core) == str):
+            self.btype = hash
+            self.bcontent = core
+            self.hash = None
+            self.core = None
+        else:
+            self.btype = ''
+            self.bcontent = ''
+            self.hash = hash
+            self.core = core
+
+        # initialize variables
+        self.valid = True
+        self.raw = None
+        self.powHash = None
+        self.powToken = None
+        self.signed = False
+        self.signature = None
+        self.signedData = None
+        self.blockFile = None
+        self.bheader = {}
+        self.bmetadata = {}
+
+        # handle arguments
+        if self.getCore() is None:
+            self.core = onionrcore.Core()
+        if not self.getHash() is None:
+            self.update()
+
+    # logic
+
+    def update(self, data = None, file = None):
+        '''
+            Loads data from a block in to the current object.
+
+            Inputs:
+            - data (str):
+              - if None: will load from file by hash
+              - else: will load from `data` string
+            - file (str):
+              - if None: will load from file specified in this parameter
+              - else: will load from wherever block is stored by hash
+
+            Outputs:
+            - (bool): indicates whether or not the operation was successful
+        '''
+
+        try:
+            # import from string
+            blockdata = data
+
+            # import from file
+            if blockdata is None:
+                filelocation = file
+
+                if filelocation is None:
+                    if self.getHash() is None:
+                        return False
+
+                    filelocation = 'data/blocks/%s.dat' % self.getHash()
+
+                blockdata = open(filelocation, 'rb').read().decode('utf-8')
+
+                self.blockFile = filelocation
+            else:
+                self.blockFile = None
+
+            # parse block
+            self.raw = str(blockdata)
+            self.bheader = json.loads(self.getRaw()[:self.getRaw().index('\n')])
+            self.bcontent = self.getRaw()[self.getRaw().index('\n') + 1:]
+            self.bmetadata = json.loads(self.getHeader('meta'))
+            self.btype = self.getMetadata('type')
+            self.powHash = self.getMetadata('powHash')
+            self.powToken = self.getMetadata('powToken')
+            self.signed = ('sig' in self.getHeader() and self.getHeader('sig') != '')
+            self.signature = (None if not self.isSigned() else self.getHeader('sig'))
+            self.signedData = (None if not self.isSigned() else self.getHeader('meta') + '\n' + self.getContent())
+            self.date = self.getCore().getBlockDate(self.getHash())
+
+            if not self.getDate() is None:
+                self.date = datetime.datetime.fromtimestamp(self.getDate())
+
+            self.valid = True
+            return True
+        except Exception as e:
+            logger.error('Failed to update block data.', error = e, timestamp = False)
+
+        self.valid = False
+        return False
+
+    def delete(self):
+        '''
+            Deletes the block's file and records, if they exist
+
+            Outputs:
+            - (bool): whether or not the operation was successful
+        '''
+
+        if self.exists():
+            os.remove(self.getBlockFile())
+            removeBlock(self.getHash())
+            return True
+        return False
+
+    def save(self, sign = False, recreate = True):
+        '''
+            Saves a block to file and imports it into Onionr
+
+            Inputs:
+            - sign (bool): whether or not to sign the block before saving
+            - recreate (bool): if the block already exists, whether or not to recreate the block and save under a new hash
+
+            Outputs:
+            - (bool): whether or not the operation was successful
+        '''
+
+        try:
+            if self.isValid() is True:
+                if (not self.getBlockFile() is None) and (recreate is True):
+                    with open(self.getBlockFile(), 'wb') as blockFile:
+                        blockFile.write(self.getRaw().encode())
+                    self.update()
+                else:
+                    self.hash = self.getCore().insertBlock(self.getContent(), header = self.getType(), sign = sign)
+                    self.update()
+                return True
+            else:
+                logger.warn('Not writing block; it is invalid.')
+        except Exception as e:
+            logger.error('Failed to save block.', error = e, timestamp = False)
+            return False
+
+    # getters
+
+    def getHash(self):
+        '''
+            Returns the hash of the block if saved to file
+
+            Outputs:
+            - (str): the hash of the block, or None
+        '''
+
+        return self.hash
+
+    def getCore(self):
+        '''
+            Returns the Core instance being used by the Block
+
+            Outputs:
+            - (Core): the Core instance
+        '''
+
+        return self.core
+
+    def getType(self):
+        '''
+            Returns the type of the block
+
+            Outputs:
+            - (str): the type of the block
+        '''
+
+        return self.btype
+
+    def getRaw(self):
+        '''
+            Returns the raw contents of the block, if saved to file
+
+            Outputs:
+            - (str): the raw contents of the block, or None
+        '''
+
+        return str(self.raw)
+
+    def getHeader(self, key = None):
+        '''
+            Returns the header information
+
+            Inputs:
+            - key (str): only returns the value of the key in the header
+
+            Outputs:
+            - (dict/str): either the whole header as a dict, or one value
+        '''
+
+        if not key is None:
+            return self.getHeader()[key]
+        else:
+            return self.bheader
+
+    def getMetadata(self, key = None):
+        '''
+            Returns the metadata information
+
+            Inputs:
+            - key (str): only returns the value of the key in the metadata
+
+            Outputs:
+            - (dict/str): either the whole metadata as a dict, or one value
+        '''
+
+        if not key is None:
+            return self.getMetadata()[key]
+        else:
+            return self.bmetadata
+
+    def getContent(self):
+        '''
+            Returns the contents of the block
+
+            Outputs:
+            - (str): the contents of the block
+        '''
+
+        return str(self.bcontent)
+
+    def getDate(self):
+        '''
+            Returns the date that the block was received, if loaded from file
+
+            Outputs:
+            - (datetime): the date that the block was received
+        '''
+
+        return self.date
+
+    def getBlockFile(self):
+        '''
+            Returns the location of the block file if it is saved
+
+            Outputs:
+            - (str): the location of the block file, or None
+        '''
+
+        return self.blockFile
+
+    def isValid(self):
+        '''
+            Checks if the block is valid
+
+            Outputs:
+            - (bool): whether or not the block is valid
+        '''
+
+        return self.valid
+
+    def isSigned(self):
+        '''
+            Checks if the block was signed
+
+            Outputs:
+            - (bool): whether or not the block is signed
+        '''
+
+        return self.signed
+
+    def getSignature(self):
+        '''
+            Returns the base64-encoded signature
+
+            Outputs:
+            - (str): the signature, or None
+        '''
+
+        return self.signature
+
+    def getSignedData(self):
+        '''
+            Returns the data that was signed
+
+            Outputs:
+            - (str): the data that was signed, or None
+        '''
+
+        return self.signedData
+
+    def isSigner(self, signer, encodedData = True):
+        '''
+            Checks if the block was signed by the signer inputted
+
+            Inputs:
+            - signer (str): the public key of the signer to check against
+            - encodedData (bool): whether or not the `signer` argument is base64 encoded
+
+            Outputs:
+            - (bool): whether or not the signer of the block is the signer inputted
+        '''
+
+        try:
+            if (not self.isSigned()) or (not self.getCore()._utils.validatePubKey(signer)):
+                return False
+
+            return bool(self.getCore()._crypto.edVerify(self.getSignedData(), signer, self.getSignature(), encodedData = encodedData))
+        except:
+            return False
+
+    # setters
+
+    def setType(self, btype):
+        '''
+            Sets the type of the block
+
+            Inputs:
+            - btype (str): the type of block to be set to
+
+            Outputs:
+            - (Block): the block instance
+        '''
+
+        self.btype = btype
+        return self
+
+    def setContent(self, bcontent):
+        '''
+            Sets the contents of the block
+
+            Inputs:
+            - bcontent (str): the contents to be set to
+
+            Outputs:
+            - (Block): the block instance
+        '''
+
+        self.bcontent = str(bcontent)
+        return self
+
+    # static
+
+    def getBlocks(type = None, signer = None, signed = None, reverse = False, core = None):
+        '''
+            Returns a list of Block objects based on supplied filters
+
+            Inputs:
+            - type (str): filters by block type
+            - signer (str/list): filters by signer (one in the list has to be a signer)
+            - signed (bool): filters out by whether or not the block is signed
+            - reverse (bool): reverses the list if True
+            - core (Core): lets you optionally supply a core instance so one doesn't need to be started
+
+            Outputs:
+            - (list): a list of Block objects that match the input
+        '''
+
+        try:
+            core = (core if not core is None else onionrcore.Core())
+
+            relevant_blocks = list()
+            blocks = (core.getBlockList() if type is None else core.getBlocksByType(type))
+
+            for block in blocks:
+                if Block.exists(block):
+                    block = Block(block, core = core)
+
+                    relevant = True
+
+                    if (not signed is None) and (block.isSigned() != bool(signed)):
+                        relevant = False
+                    if not signer is None:
+                        if isinstance(signer, (str,)):
+                            signer = [signer]
+
+                        isSigner = False
+                        for key in signer:
+                            if block.isSigner(key):
+                                isSigner = True
+                                break
+
+                        if not isSigner:
+                            relevant = False
+
+                    if relevant:
+                        relevant_blocks.append(block)
+
+            if bool(reverse):
+                relevant_blocks.reverse()
+
+            return relevant_blocks
+        except Exception as e:
+            logger.debug(('Failed to get blocks: %s' % str(e)) + logger.parse_error())
+
+        return list()
+
+    def exists(hash):
+        '''
+            Checks if a block is saved to file or not
+
+            Inputs:
+            - hash (str/Block):
+              - if (Block): check if this block is saved to file
+              - if (str): check if a block by this hash is in file
+
+            Outputs:
+            - (bool): whether or not the block file exists
+        '''
+
+        if hash is None:
+            return False
+        elif type(hash) == Block:
+            blockfile = hash.getBlockFile()
+        else:
+            blockfile = 'data/blocks/%s.dat' % hash
+
+        return os.path.exists(blockfile) and os.path.isfile(blockfile)

onionr/onionrblocks.py

@@ -1,26 +0,0 @@
-'''
-Onionr - P2P Microblogging Platform & Social network.
-
-This class contains the OnionrBlocks class which is a class for working with Onionr blocks
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import json
-class OnionrBlocks:
-    def __init__(self, coreInstance):
-        return
-    def metadataGenerate(self):
-        return
-    

onionr/onionrcrypto.py

@@ -17,15 +17,19 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import nacl.signing, nacl.encoding, nacl.public, nacl.secret, os, binascii, base64, hashlib, logger
+import nacl.signing, nacl.encoding, nacl.public, nacl.hash, nacl.secret, os, binascii, base64, hashlib, logger, onionrproofs, time, math
 
 class OnionrCrypto:
     def __init__(self, coreInstance):
         self._core = coreInstance
         self._keyFile = 'data/keys.txt'
+        self.keyPowFile = 'data/keyPow.txt'
         self.pubKey = None
         self.privKey = None
 
+        self.pubKeyPowToken = None
+        #self.pubKeyPowHash = None
+
         self.HASH_ID_ROUNDS = 2000
 
         # Load our own pub/priv Ed25519 keys, gen & save them if they don't exist
@@ -34,12 +38,29 @@ class OnionrCrypto:
                 keys = keys.read().split(',')
                 self.pubKey = keys[0]
                 self.privKey = keys[1]
+            try:
+                with open(self.keyPowFile, 'r') as powFile:
+                    data = powFile.read()
+                    self.pubKeyPowToken = data
+            except (FileNotFoundError, IndexError):
+                pass
         else:
             keys = self.generatePubKey()
             self.pubKey = keys[0]
             self.privKey = keys[1]
             with open(self._keyFile, 'w') as keyfile:
                 keyfile.write(self.pubKey + ',' + self.privKey)
+            with open(self.keyPowFile, 'w') as keyPowFile:
+                proof = onionrproofs.POW(self.pubKey)
+                logger.info('Doing necessary work to insert our public key')
+                while True:
+                    time.sleep(0.2)
+                    powToken = proof.getResult()
+                    if powToken != False:
+                        break
+                keyPowFile.write(base64.b64encode(powToken[1]).decode())
+                self.pubKeyPowToken = powToken[1]
+                self.pubKeyPowHash = powToken[0]
         return
 
     def edVerify(self, data, key, sig, encodedData=True):
@@ -60,7 +81,6 @@ class OnionrCrypto:
                 retData = key.verify(data, sig) # .encode() is not the same as nacl.encoding
             except nacl.exceptions.BadSignatureError:
                 pass
-
         else:
             try:
                 retData = key.verify(data, sig)
@@ -138,8 +158,6 @@ class OnionrCrypto:
         decrypted = self.symmetricDecrypt(data, key, encodedKey=True)
         return decrypted
 
-        return
-
     def symmetricEncrypt(self, data, key, encodedKey=False, returnEncoded=True):
         '''Encrypt data to a 32-byte key (Salsa20-Poly1305 MAC)'''
         if encodedKey:
@@ -210,3 +228,48 @@ class OnionrCrypto:
             prev = hasher.hexdigest()
         result = prev
         return result
+
+    def sha3Hash(self, data):
+        hasher = hashlib.sha3_256()
+        hasher.update(data)
+        return hasher.hexdigest()
+
+    def blake2bHash(self, data):
+        try:
+            data = data.encode()
+        except AttributeError:
+            pass
+        return nacl.hash.blake2b(data)
+
+    def verifyPow(self, blockContent, metadata):
+        '''
+            Verifies the proof of work associated with a block
+        '''
+        retData = False
+
+        if not (('powToken' in metadata) and ('powHash' in metadata)):
+            return False
+
+        dataLen = len(blockContent)
+
+        expectedHash = self.blake2bHash(base64.b64decode(metadata['powToken']) + self.blake2bHash(blockContent.encode()))
+        difficulty = 0
+        try:
+            expectedHash = expectedHash.decode()
+        except AttributeError:
+            pass
+        if metadata['powHash'] == expectedHash:
+            difficulty = math.floor(dataLen / 1000000)
+
+            mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode()
+            puzzle = mainHash[:difficulty]
+
+            if metadata['powHash'][:difficulty] == puzzle:
+                # logger.debug('Validated block pow')
+                retData = True
+            else:
+                logger.debug("Invalid token (#1)")
+        else:
+            logger.debug('Invalid token (#2): Expected hash %s, got hash %s...' % (metadata['powHash'], expectedHash))
+
+        return retData

onionr/onionrpluginapi.py

@@ -18,7 +18,7 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 
-import onionrplugins as plugins, logger
+import onionrplugins, core as onionrcore, logger
 
 class DaemonAPI:
     def __init__(self, pluginapi):
@@ -40,9 +40,7 @@ class DaemonAPI:
         return
 
     def local_command(self, command):
-        self.pluginapi.get_utils().localCommand(self, command)
-
-        return
+        return self.pluginapi.get_utils().localCommand(self, command)
 
     def queue_pop(self):
         return self.get_core().daemonQueue()
@@ -52,34 +50,34 @@ class PluginAPI:
         self.pluginapi = pluginapi
 
     def start(self, name):
-        plugins.start(name)
+        onionrplugins.start(name)
 
     def stop(self, name):
-        plugins.stop(name)
+        onionrplugins.stop(name)
 
     def reload(self, name):
-        plugins.reload(name)
+        onionrplugins.reload(name)
 
     def enable(self, name):
-        plugins.enable(name)
+        onionrplugins.enable(name)
 
     def disable(self, name):
-        plugins.disable(name)
+        onionrplugins.disable(name)
 
     def event(self, name, data = {}):
         events.event(name, data = data, onionr = self.pluginapi.get_onionr())
 
     def is_enabled(self, name):
-        return plugins.is_enabled(name)
+        return onionrplugins.is_enabled(name)
 
     def get_enabled_plugins(self):
-        return plugins.get_enabled()
+        return onionrplugins.get_enabled()
 
     def get_folder(self, name = None, absolute = True):
-        return plugins.get_plugins_folder(name = name, absolute = absolute)
+        return onionrplugins.get_plugins_folder(name = name, absolute = absolute)
 
     def get_data_folder(self, name, absolute = True):
-        return plugins.get_plugin_data_folder(name, absolute = absolute)
+        return onionrplugins.get_plugin_data_folder(name, absolute = absolute)
 
     def daemon_event(self, event, plugin = None):
         return # later make local command like /client/?action=makeEvent&event=eventname&module=modulename
@@ -136,6 +134,10 @@ class pluginapi:
     def __init__(self, onionr, data):
         self.onionr = onionr
         self.data = data
+        if self.onionr is None:
+            self.core = onionrcore.Core()
+        else:
+            self.core = self.onionr.onionrCore
 
         self.daemon = DaemonAPI(self)
         self.plugins = PluginAPI(self)
@@ -148,10 +150,13 @@ class pluginapi:
         return self.data
 
     def get_core(self):
-        return self.get_onionr().onionrCore
+        return self.core
 
     def get_utils(self):
-        return self.get_onionr().onionrUtils
+        return self.get_core()._utils
+
+    def get_crypto(self):
+        return self.get_core()._crypto
 
     def get_daemonapi(self):
         return self.daemon

onionr/onionrplugins.py

@@ -62,17 +62,20 @@ def enable(name, onionr = None, start_event = True):
 
     if exists(name):
         enabled_plugins = get_enabled_plugins()
-        enabled_plugins.append(name)
-        config_plugins = config.get('plugins')
-        config_plugins['enabled'] = enabled_plugins
-        config.set('plugins', config_plugins, True)
+        if not name in enabled_plugins:
+            enabled_plugins.append(name)
+            config_plugins = config.get('plugins')
+            config_plugins['enabled'] = enabled_plugins
+            config.set('plugins', config_plugins, True)
 
-        events.call(get_plugin(name), 'enable', onionr)
+            events.call(get_plugin(name), 'enable', onionr)
 
-        if start_event is True:
-            start(name)
+            if start_event is True:
+                start(name)
 
-        return True
+            return True
+        else:
+            return False
     else:
         logger.error('Failed to enable plugin \"' + name + '\", disabling plugin.')
         disable(name)

onionr/onionrproofs.py

@@ -18,8 +18,8 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 
-import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, logger
-import btc
+import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, logger, sys
+import core
 
 class POW:
     def pow(self, reporting = False):
@@ -30,20 +30,10 @@ class POW:
         answer = ''
         heartbeat = 200000
         hbCount = 0
-        blockCheck = 300000 # How often the hasher should check if the bitcoin block is updated (slows hashing but prevents less wasted work)
-        blockCheckCount = 0
-        block = '' #self.bitcoinNode.getBlockHash(self.bitcoinNode.getLastBlockHeight())
+        myCore = core.Core()
         while self.hashing:
-            '''
-            if blockCheckCount == blockCheck:
-                if self.reporting:
-                    logger.debug('Refreshing Bitcoin block')
-                block = '' #self.bitcoinNode.getBlockHash(self.bitcoinNode.getLastBlockHeight())
-                blockCheckCount = 0
-            blockCheckCount += 1
-            hbCount += 1
-            '''
-            token = nacl.hash.blake2b(nacl.utils.random()).decode()
+            rand = nacl.utils.random()
+            token = nacl.hash.blake2b(rand + self.data).decode()
             #print(token)
             if self.puzzle == token[0:self.difficulty]:
                 self.hashing = False
@@ -56,17 +46,28 @@ class POW:
             if self.reporting:
                 logger.info('Found token ' + token, timestamp=True)
                 logger.info('took ' + str(endTime - startTime) + ' seconds', timestamp=True)
-            self.result = token
+            self.result = (token, rand)
 
-    def __init__(self, difficulty, bitcoinNode=''):
+    def __init__(self, data):
         self.foundHash = False
-        self.difficulty = difficulty
+        self.difficulty = 0
+        self.data = data
+
+        dataLen = sys.getsizeof(data)
+        self.difficulty = math.floor(dataLen/1000000)
+        if self.difficulty <= 2:
+            self.difficulty = 4
+
+        try:
+            self.data = self.data.encode()
+        except AttributeError:
+            pass
+        self.data = nacl.hash.blake2b(self.data)
 
         logger.debug('Computing difficulty of ' + str(self.difficulty))
 
         self.mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode()
         self.puzzle = self.mainHash[0:self.difficulty]
-        self.bitcoinNode = bitcoinNode
         #logger.debug('trying to find ' + str(self.mainHash))
         tOne = threading.Thread(name='one', target=self.pow, args=(True,))
         tTwo = threading.Thread(name='two', target=self.pow, args=(True,))

onionr/onionrutils.py

@@ -18,7 +18,7 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 # Misc functions that do not fit in the main api, but are useful
-import getpass, sys, requests, os, socket, hashlib, logger, sqlite3, config, binascii, time, base64, json
+import getpass, sys, requests, os, socket, hashlib, logger, sqlite3, config, binascii, time, base64, json, glob, shutil, math
 import nacl.signing, nacl.encoding
 
 if sys.version_info < (3, 6):
@@ -71,7 +71,7 @@ class OnionrUtils:
             if block == '':
                 logger.error('Could not send PM')
             else:
-                logger.info('Sent PM, hash: ' + block)
+                logger.info('Sent PM, hash: %s' % block)
         except Exception as error:
             logger.error('Failed to send PM.', error=error)
 
@@ -101,9 +101,26 @@ class OnionrUtils:
             retVal = False
             if newKeyList != False:
                 for key in newKeyList.split(','):
-                    if not key in self._core.listPeers(randomOrder=False) and type(key) != None and key != self._core._crypto.pubKey:
-                        if self._core.addPeer(key):
-                            retVal = True
+                    key = key.split('-')
+                    try:
+                        if len(key[0]) > 60 or len(key[1]) > 1000:
+                            logger.warn('%s or its pow value is too large.' % key[0])
+                            continue
+                    except IndexError:
+                        logger.warn('No pow token')
+                        continue
+                    powHash = self._core._crypto.blake2bHash(base64.b64decode(key[1]) + self._core._crypto.blake2bHash(key[0].encode()))
+                    try:
+                        powHash = powHash.encode()
+                    except AttributeError:
+                        pass
+                    if powHash.startswith(b'0000'):
+                        if not key[0] in self._core.listPeers(randomOrder=False) and type(key) != None and key[0] != self._core._crypto.pubKey:
+                            if self._core.addPeer(key[0], key[1]):
+                                retVal = True
+                    else:
+                        logger.warn(powHash)
+                        logger.warn('%s pow failed' % key[0])
             return retVal
         except Exception as error:
             logger.error('Failed to merge keys.', error=error)
@@ -118,15 +135,15 @@ class OnionrUtils:
             retVal = False
             if newAdderList != False:
                 for adder in newAdderList.split(','):
-                    if not adder in self._core.listAdders(randomOrder=False) and adder.strip() != self.getMyAddress():
+                    if not adder in self._core.listAdders(randomOrder = False) and adder.strip() != self.getMyAddress():
                         if self._core.addAddress(adder):
-                            logger.info('Added ' + adder + ' to db.', timestamp=True)
+                            logger.info('Added %s to db.' % adder, timestamp = True)
                             retVal = True
                     else:
-                        logger.debug(adder + " is either our address or already in our DB")
+                        logger.debug('%s is either our address or already in our DB' % adder)
             return retVal
         except Exception as error:
-            logger.error('Failed to merge adders.', error=error)
+            logger.error('Failed to merge adders.', error = error)
             return False
 
     def getMyAddress(self):
@@ -134,7 +151,7 @@ class OnionrUtils:
             with open('./data/hs/hostname', 'r') as hostname:
                 return hostname.read().strip()
         except Exception as error:
-            logger.error('Failed to read my address.', error=error)
+            logger.error('Failed to read my address.', error = error)
             return None
 
     def localCommand(self, command, silent = True):
@@ -149,7 +166,7 @@ class OnionrUtils:
             retData = requests.get('http://' + open('data/host.txt', 'r').read() + ':' + str(config.get('client')['port']) + '/client/?action=' + command + '&token=' + str(config.get('client')['client_hmac']) + '&timingToken=' + self.timingToken).text
         except Exception as error:
             if not silent:
-                logger.error('Failed to make local request (command: ' + str(command) + ').', error=error)
+                logger.error('Failed to make local request (command: %s).' % command, error=error)
             retData = False
 
         return retData
@@ -327,7 +344,7 @@ class OnionrUtils:
         '''
             Find, decrypt, and return array of PMs (array of dictionary, {from, text})
         '''
-        #blocks = self._core.getBlockList().split('\n')
+        #blocks = self._core.getBlockList()
         blocks = self._core.getBlocksByType('pm')
         message = ''
         sender = ''
@@ -336,52 +353,43 @@ class OnionrUtils:
                 continue
             try:
                 with open('data/blocks/' + i + '.dat', 'r') as potentialMessage:
-                    data = potentialMessage.read().split('}')
-                    message = data[1]
-                    sigResult = ''
-                    signer = ''
+                    potentialMessage = potentialMessage.read()
+                    blockMetadata = json.loads(potentialMessage[:potentialMessage.find('\n')])
+                    blockContent = potentialMessage[potentialMessage.find('\n') + 1:]
 
                     try:
-                        metadata = json.loads(data[0] + '}')
-                    except json.decoder.JSONDecodeError:
-                        metadata = {}
-                    '''
-                    sigResult = self._core._crypto.edVerify(message, signer, sig, encodedData=True)
-                    #sigResult = False
-                    if sigResult != False:
-                        sigResult = 'Valid signature by ' + signer
-                    else:
-                        sigResult = 'Invalid signature by ' + signer
-                    '''
-
-                    try:
-                        message = self._core._crypto.pubKeyDecrypt(message, encodedData=True, anonymous=True)
+                        message = self._core._crypto.pubKeyDecrypt(blockContent, encodedData=True, anonymous=True)
                     except nacl.exceptions.CryptoError as e:
-                        logger.error('Unable to decrypt ' + i, error=e)
+                        pass
                     else:
                         try:
-                            message = json.loads(message.decode())
-                            message['msg']
-                            message['id']
-                            message['sig']
+                            message = message.decode()
+                        except AttributeError:
+                            pass
+
+                        try:
+                            message = json.loads(message)
                         except json.decoder.JSONDecodeError:
-                            logger.error('Could not decode PM JSON')
-                        except KeyError:
-                            logger.error('PM is missing JSON keys')
+                            pass
                         else:
-                            if self.validatePubKey(message['id']):
-                                sigResult = self._core._crypto.edVerify(message['msg'], message['id'], message['sig'], encodedData=True)
-                                logger.info('-----------------------------------')
-                                logger.info('Recieved message: ' + message['msg'])
-                                if sigResult:
-                                    logger.info('Valid signature by ' + message['id'])
+                            logger.info('Decrypted %s:' % i)
+                            logger.info(message["msg"])
+
+                            signer = message["id"]
+                            sig = message["sig"]
+
+                            if self.validatePubKey(signer):
+                                if self._core._crypto.edVerify(message["msg"], signer, sig, encodedData=True):
+                                    logger.info("Good signature by %s" % signer)
                                 else:
-                                    logger.warn('Invalid signature by ' + message['id'])
+                                    logger.warn("Bad signature by %s" % signer)
+                            else:
+                                logger.warn('Bad sender id: %s' % signer)
 
             except FileNotFoundError:
                 pass
             except Exception as error:
-                logger.error('Failed to open block ' + str(i) + '.', error=error)
+                logger.error('Failed to open block %s.' % i, error=error)
         return
 
     def getPeerByHashId(self, hash):
@@ -423,4 +431,74 @@ class OnionrUtils:
             return False
 
     def token(self, size = 32):
+        '''
+            Generates a secure random hex encoded token
+        '''
         return binascii.hexlify(os.urandom(size))
+
+    def importNewBlocks(self, scanDir=''):
+        '''
+            This function is intended to scan for new blocks ON THE DISK and import them
+        '''
+        blockList = self._core.getBlockList()
+        if scanDir == '':
+            scanDir = self._core.blockDataLocation
+        if not scanDir.endswith('/'):
+            scanDir += '/'
+        for block in glob.glob(scanDir + "*.dat"):
+            if block.replace(scanDir, '').replace('.dat', '') not in blockList:
+                logger.info('Found new block on dist %s' % block)
+                with open(block, 'rb') as newBlock:
+                    block = block.replace(scanDir, '').replace('.dat', '')
+                    if self._core._crypto.sha3Hash(newBlock.read()) == block.replace('.dat', ''):
+                        self._core.addToBlockDB(block.replace('.dat', ''), dataSaved=True)
+                        logger.info('Imported block %s.' % block)
+                    else:
+                        logger.warn('Failed to verify hash for %s' % block)
+
+    def progressBar(self, value = 0, endvalue = 100, width = None):
+        '''
+            Outputs a progress bar with a percentage. Write \n after use.
+        '''
+
+        if width is None or height is None:
+            width, height = shutil.get_terminal_size((80, 24))
+
+        bar_length = width - 6
+
+        percent = float(value) / endvalue
+        arrow = '─' * int(round(percent * bar_length)-1) + '>'
+        spaces = ' ' * (bar_length - len(arrow))
+
+        sys.stdout.write("\r┣{0}┫ {1}%".format(arrow + spaces, int(round(percent * 100))))
+        sys.stdout.flush()
+    
+    def getEpoch(self):
+        '''returns epoch'''
+        return math.floor(time.time())
+
+def size(path='.'):
+    '''
+        Returns the size of a folder's contents in bytes
+    '''
+    total = 0
+    if os.path.exists(path):
+        if os.path.isfile(path):
+            total = os.path.getsize(path)
+        else:
+            for entry in os.scandir(path):
+                if entry.is_file():
+                    total += entry.stat().st_size
+                elif entry.is_dir():
+                    total += size(entry.path)
+    return total
+
+def humanSize(num, suffix='B'):
+    '''
+        Converts from bytes to a human readable format.
+    '''
+    for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
+        if abs(num) < 1024.0:
+            return "%.1f %s%s" % (num, unit, suffix)
+        num /= 1024.0
+    return "%.1f %s%s" % (num, 'Yi', suffix)

onionr/static-data/default-plugins/gui/info.json

@@ -0,0 +1,5 @@
+{
+    "name" : "gui",
+    "version" : "1.0",
+    "author" : "onionr"
+}

onionr/static-data/default-plugins/gui/main.py

@@ -16,46 +16,54 @@
 '''
 
 # Imports some useful libraries
-import logger, config
-import os, sqlite3, core
+import logger, config, core
+import os, sqlite3, threading
+from onionrblockapi import Block
+
+plugin_name = 'gui'
+
+def send():
+    global message
+    block = Block()
+    block.setType('txt')
+    block.setContent(message)
+    logger.debug('Sent message in block %s.' % block.save(sign = True))
+
 
 def sendMessage():
     global sendEntry
 
-    messageToAdd = '-txt-' + sendEntry.get()
-    #addedHash = pluginapi.get_core().setData(messageToAdd)
-    #pluginapi.get_core().addToBlockDB(addedHash, selfInsert=True)
-    #pluginapi.get_core().setBlockType(addedHash, 'txt')
-    pluginapi.get_core().insertBlock(messageToAdd, header='txt', sign=True)
-    sendEntry.delete(0, END)
+    global message
+    message = sendEntry.get()
+
+    t = threading.Thread(target = send)
+    t.start()
+
+    sendEntry.delete(0, len(message))
 
 def update():
     global listedBlocks, listbox, runningCheckDelayCount, runningCheckDelay, root, daemonStatus
 
-    # TO DO: migrate to new header format
-    for i in pluginapi.get_core().getBlocksByType('txt'):
-        if i.strip() == '' or i in listedBlocks:
+    for i in Block.getBlocks(type = 'txt'):
+        if i.getContent().strip() == '' or i.getHash() in listedBlocks:
             continue
-        blockFile = open('./data/blocks/' + i + '.dat')
-        listbox.insert(END, str(blockFile.read().replace('-txt-', '')))
-        blockFile.close()
-        listedBlocks.append(i)
-        listbox.see(END)
-    blocksList = os.listdir('./data/blocks/') # dir is your directory path
-    number_blocks = len(blocksList)
+        listbox.insert(99999, str(i.getContent()))
+        listedBlocks.append(i.getHash())
+        listbox.see(99999)
+
     runningCheckDelayCount += 1
 
     if runningCheckDelayCount == runningCheckDelay:
-        resp = pluginapi.get_core()._utils.localCommand('ping')
+        resp = pluginapi.daemon.local_command('ping')
         if resp == 'pong':
-            daemonStatus.config(text="Onionr Daemon Status: Running")
+            daemonStatus.config(text = "Onionr Daemon Status: Running")
         else:
-            daemonStatus.config(text="Onionr Daemon Status: Not Running")
+            daemonStatus.config(text = "Onionr Daemon Status: Not Running")
         runningCheckDelayCount = 0
     root.after(10000, update)
 
 
-def openGUI():
+def reallyOpenGUI():
     import tkinter
     global root, runningCheckDelay, runningCheckDelayCount, scrollbar, listedBlocks, nodeInfo, keyInfo, idText, idEntry, pubKeyEntry, listbox, daemonStatus, sendEntry
 
@@ -74,11 +82,12 @@ def openGUI():
     nodeInfo = tkinter.Frame(root)
     keyInfo = tkinter.Frame(root)
 
-    print(pluginapi.get_onionr().get_hostname())
-    idText = pluginapi.get_onionr().get_hostname()
+    hostname = pluginapi.get_onionr().get_hostname()
+    logger.debug('Onionr Hostname: %s' % hostname)
+    idText = hostname
 
     idEntry = tkinter.Entry(nodeInfo)
-    tkinter.Label(nodeInfo, text="Node Address: ").pack(side=tkinter.LEFT)
+    tkinter.Label(nodeInfo, text = "Node Address: ").pack(side=tkinter.LEFT)
     idEntry.pack()
     idEntry.insert(0, idText.strip())
     idEntry.configure(state="readonly")
@@ -100,17 +109,22 @@ def openGUI():
     sendEntry.pack(side=tkinter.TOP, pady=5)
     sendBtn.pack(side=tkinter.TOP)
 
-    listbox = tkinter.Listbox(root, yscrollcommand=tkinter.scrollbar.set, height=15)
+    listbox = tkinter.Listbox(root, yscrollcommand=tkinter.Scrollbar.set, height=15)
 
     listbox.pack(fill=tkinter.BOTH, pady=25)
 
     daemonStatus = tkinter.Label(root, text="Onionr Daemon Status: unknown")
     daemonStatus.pack()
 
-    scrollbar.config(command=tkinter.listbox.yview)
+    scrollbar.config(command=tkinter.Listbox.yview)
     root.after(2000, update)
     root.mainloop()
 
+def openGUI():
+    t = threading.Thread(target = reallyOpenGUI)
+    t.daemon = False
+    t.start()
+
 def on_init(api, data = None):
     global pluginapi
     pluginapi = api

onionr/static-data/default-plugins/pluginmanager/info.json

@@ -0,0 +1,5 @@
+{
+    "name" : "pluginmanager",
+    "version" : "1.0",
+    "author" : "onionr"
+}

onionr/static-data/default-plugins/pluginmanager/main.py

@@ -0,0 +1,546 @@
+'''
+    Onionr - P2P Microblogging Platform & Social network.
+
+    This plugin acts as a plugin manager, and allows the user to install other plugins distributed over Onionr.
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+# useful libraries
+import logger, config
+import os, sys, json, time, random, shutil, base64, getpass, datetime, re
+from onionrblockapi import Block
+
+plugin_name = 'pluginmanager'
+
+keys_data = {'keys' : {}, 'plugins' : [], 'repositories' : {}}
+
+# key functions
+
+def writeKeys():
+    '''
+        Serializes and writes the keystore in memory to file
+    '''
+
+    file = open(keys_file, 'w')
+    file.write(json.dumps(keys_data, indent=4, sort_keys=True))
+    file.close()
+
+def readKeys():
+    '''
+        Loads the keystore into memory
+    '''
+
+    global keys_data
+    keys_data = json.loads(open(keys_file).read())
+    return keys_data
+
+def getKey(plugin):
+    '''
+        Returns the public key for a given plugin
+    '''
+
+    global keys_data
+    readKeys()
+    return (keys_data['keys'][plugin] if plugin in keys_data['keys'] else None)
+
+def saveKey(plugin, key):
+    '''
+        Saves the public key for a plugin to keystore
+    '''
+
+    global keys_data
+    readKeys()
+    keys_data['keys'][plugin] = key
+    writeKeys()
+
+def getPlugins():
+    '''
+        Returns a list of plugins installed by the plugin manager
+    '''
+
+    global keys_data
+    readKeys()
+    return keys_data['plugins']
+
+def addPlugin(plugin):
+    '''
+        Saves the plugin name, to remember that it was installed by the pluginmanager
+    '''
+
+    global keys_data
+    readKeys()
+    if not plugin in keys_data['plugins']:
+        keys_data['plugins'].append(plugin)
+        writeKeys()
+
+def removePlugin(plugin):
+    '''
+        Removes the plugin name from the pluginmanager's records
+    '''
+
+    global keys_data
+    readKeys()
+    if plugin in keys_data['plugins']:
+        keys_data['plugins'].remove(plugin)
+        writeKeys()
+
+def getRepositories():
+    '''
+        Returns a list of plugins installed by the plugin manager
+    '''
+
+    global keys_data
+    readKeys()
+    return keys_data['repositories']
+
+def addRepository(repositories, data):
+    '''
+        Saves the plugin name, to remember that it was installed by the pluginmanager
+    '''
+
+    global keys_data
+    readKeys()
+    keys_data['repositories'][repositories] = data
+    writeKeys()
+
+def removeRepository(repositories):
+    '''
+        Removes the plugin name from the pluginmanager's records
+    '''
+
+    global keys_data
+    readKeys()
+    if plugin in keys_data['repositories']:
+        del keys_data['repositories'][repositories]
+        writeKeys()
+
+def check():
+    '''
+        Checks to make sure the keystore file still exists
+    '''
+
+    global keys_file
+    keys_file = pluginapi.plugins.get_data_folder(plugin_name) + 'keystore.json'
+    if not os.path.isfile(keys_file):
+        writeKeys()
+
+# plugin management
+
+def sanitize(name):
+    return re.sub('[^0-9a-zA-Z]+', '', str(name).lower())[:255]
+
+def blockToPlugin(block):
+    try:
+        block = Block(block)
+        blockContent = json.loads(block.getContent())
+
+        name = sanitize(blockContent['name'])
+        author = blockContent['author']
+        date = blockContent['date']
+        version = None
+
+        if 'version' in blockContent['info']:
+            version = blockContent['info']['version']
+
+        content = base64.b64decode(blockContent['content'].encode())
+
+        source = pluginapi.plugins.get_data_folder(plugin_name) + 'plugin.zip'
+        destination = pluginapi.plugins.get_folder(name)
+
+        with open(source, 'wb') as f:
+            f.write(content)
+
+        if os.path.exists(destination) and not os.path.isfile(destination):
+            shutil.rmtree(destination)
+
+        shutil.unpack_archive(source, destination)
+        pluginapi.plugins.enable(name)
+
+        logger.info('Installation of %s complete.' % name)
+
+        return True
+    except Exception as e:
+        logger.error('Failed to install plugin.', error = e, timestamp = False)
+
+    return False
+
+def pluginToBlock(plugin, import_block = True):
+    try:
+        plugin = sanitize(plugin)
+
+        directory = pluginapi.get_pluginapi().get_folder(plugin)
+        data_directory = pluginapi.get_pluginapi().get_data_folder(plugin)
+        zipfile = pluginapi.get_pluginapi().get_data_folder(plugin_name) + 'plugin.zip'
+
+        if os.path.exists(directory) and not os.path.isfile(directory):
+            if os.path.exists(data_directory) and not os.path.isfile(data_directory):
+                shutil.rmtree(data_directory)
+            if os.path.exists(zipfile) and os.path.isfile(zipfile):
+                os.remove(zipfile)
+            if os.path.exists(directory + '__pycache__') and not os.path.isfile(directory + '__pycache__'):
+                shutil.rmtree(directory + '__pycache__')
+
+            shutil.make_archive(zipfile[:-4], 'zip', directory)
+            data = base64.b64encode(open(zipfile, 'rb').read())
+
+            author = getpass.getuser()
+            description = 'Default plugin description'
+            info = {"name" : plugin}
+            try:
+                if os.path.exists(directory + 'info.json'):
+                    info = json.loads(open(directory + 'info.json').read())
+                    if 'author' in info:
+                        author = info['author']
+                    if 'description' in info:
+                        description = info['description']
+            except:
+                pass
+
+            metadata = {'author' : author, 'date' : str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')), 'name' : plugin, 'info' : info, 'compiled-by' : plugin_name, 'content' : data.decode('utf-8'), 'description' : description}
+
+            hash = pluginapi.get_core().insertBlock(json.dumps(metadata), header = 'plugin', sign = True)
+
+            if import_block:
+                pluginapi.get_utils().importNewBlocks()
+
+            return hash
+        else:
+            logger.error('Plugin %s does not exist.' % plugin)
+    except Exception as e:
+        logger.error('Failed to convert plugin to block.', error = e, timestamp = False)
+
+    return False
+
+def installBlock(block):
+    try:
+        block = Block(block)
+        blockContent = json.loads(block.getContent())
+
+        name = sanitize(blockContent['name'])
+        author = blockContent['author']
+        date = blockContent['date']
+        version = None
+
+        if 'version' in blockContent['info']:
+            version = blockContent['info']['version']
+
+        install = False
+
+        logger.info(('Will install %s' + (' v' + version if not version is None else '') + ' (%s), by %s') % (name, date, author))
+
+        # TODO: Convert to single line if statement
+        if os.path.exists(pluginapi.plugins.get_folder(name)):
+            install = logger.confirm(message = 'Continue with installation (will overwrite existing plugin) %s?')
+        else:
+            install = logger.confirm(message = 'Continue with installation %s?')
+
+        if install:
+            blockToPlugin(block.getHash())
+            addPlugin(name)
+        else:
+            logger.info('Installation cancelled.')
+            return False
+
+        return True
+    except Exception as e:
+        logger.error('Failed to install plugin.', error = e, timestamp = False)
+        return False
+
+def uninstallPlugin(plugin):
+    try:
+        plugin = sanitize(plugin)
+
+        pluginFolder = pluginapi.plugins.get_folder(plugin)
+        exists = (os.path.exists(pluginFolder) and not os.path.isfile(pluginFolder))
+        installedByPluginManager = plugin in getPlugins()
+        remove = False
+
+        if not exists:
+            logger.warn('Plugin %s does not exist.' % plugin, timestamp = False)
+            return False
+
+        default = 'y'
+        if not installedByPluginManager:
+            logger.warn('The plugin %s was not installed by %s.' % (plugin, plugin_name), timestamp = False)
+            default = 'n'
+        remove = logger.confirm(message = 'All plugin data will be lost. Are you sure you want to proceed %s?', default = default)
+
+        if remove:
+            if installedByPluginManager:
+                removePlugin(plugin)
+            pluginapi.plugins.disable(plugin)
+            shutil.rmtree(pluginFolder)
+
+            logger.info('Uninstallation of %s complete.' % plugin)
+
+            return True
+        else:
+            logger.info('Uninstallation cancelled.')
+    except Exception as e:
+        logger.error('Failed to uninstall plugin.', error = e)
+    return False
+
+# command handlers
+
+def help():
+    logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin> [public key/block hash]')
+    logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin> [public key/block hash]')
+
+def commandInstallPlugin():
+    if len(sys.argv) >= 3:
+        check()
+
+        pluginname = sys.argv[2]
+        pkobh = None # public key or block hash
+
+        version = None
+        if ':' in pluginname:
+            details = pluginname
+            pluginname = sanitize(details[0])
+            version = details[1]
+
+        sanitize(pluginname)
+
+        if len(sys.argv) >= 4:
+            # public key or block hash specified
+            pkobh = sys.argv[3]
+        else:
+            # none specified, check if in config file
+            pkobh = getKey(pluginname)
+
+        if pkobh is None:
+            # still nothing found, try searching repositories
+            logger.info('Searching for public key in repositories...')
+            try:
+                repos = getRepositories()
+                distributors = list()
+                for repo, records in repos.items():
+                    if pluginname in records:
+                        logger.debug('Found %s in repository %s for plugin %s.' % (records[pluginname], repo, pluginname))
+                        distributors.append(records[pluginname])
+
+                if len(distributors) != 0:
+                    distributor = None
+
+                    if len(distributors) == 1:
+                        logger.info('Found distributor: %s' % distributors[0])
+                        distributor = distributors[0]
+                    else:
+                        distributors_message = ''
+
+                        index = 1
+                        for dist in distributors:
+                            distributors_message += '    ' + logger.colors.bold + str(index) + ') ' + logger.colors.reset + str(dist) + '\n'
+                            index += 1
+
+                        logger.info((logger.colors.bold + 'Found distributors (%s):' + logger.colors.reset + '\n' + distributors_message) % len(distributors))
+
+                        valid = False
+                        while not valid:
+                            choice = logger.readline('Select the number of the key to use, from 1 to %s, or press Ctrl+C to cancel:' % (index - 1))
+
+                            try:
+                                if int(choice) < index and int(choice) >= 1:
+                                    distributor = distributors[int(choice)]
+                                    valid = True
+                            except KeyboardInterrupt:
+                                logger.info('Installation cancelled.')
+                                return True
+                            except:
+                                pass
+
+                    if not distributor is None:
+                        pkobh = distributor
+            except Exception as e:
+                logger.warn('Failed to lookup plugin in repositories.', timestamp = False)
+                logger.error('asdf', error = e, timestamp = False)
+
+        if pkobh is None:
+            logger.error('No key for this plugin found in keystore or repositories, please specify.')
+            help()
+            return True
+
+        valid_hash = pluginapi.get_utils().validateHash(pkobh)
+        real_block = False
+        valid_key = pluginapi.get_utils().validatePubKey(pkobh)
+        real_key = False
+
+        if valid_hash:
+            real_block = Block.exists(pkobh)
+        elif valid_key:
+            real_key = pluginapi.get_utils().hasKey(pkobh)
+
+        blockhash = None
+
+        if valid_hash and not real_block:
+            logger.error('Block hash not found. Perhaps it has not been synced yet?')
+            logger.debug('Is valid hash, but does not belong to a known block.')
+
+            return True
+        elif valid_hash and real_block:
+            blockhash = str(pkobh)
+            logger.debug('Using block %s...' % blockhash)
+
+            installBlock(blockhash)
+        elif valid_key and not real_key:
+            logger.error('Public key not found. Try adding the node by address manually, if possible.')
+            logger.debug('Is valid key, but the key is not a known one.')
+        elif valid_key and real_key:
+            publickey = str(pkobh)
+            logger.debug('Using public key %s...' % publickey)
+
+            saveKey(pluginname, pkobh)
+
+            signedBlocks = Block.getBlocks(type = 'plugin', signed = True, signer = publickey)
+
+            mostRecentTimestamp = None
+            mostRecentVersionBlock = None
+
+            for block in signedBlocks:
+                try:
+                    blockContent = json.loads(block.getContent())
+
+                    if not (('author' in blockContent) and ('info' in blockContent) and ('date' in blockContent) and ('name' in blockContent)):
+                        raise ValueError('Missing required parameter `date` in block %s.' % block.getHash())
+
+                    blockDatetime = datetime.datetime.strptime(blockContent['date'], '%Y-%m-%d %H:%M:%S')
+
+                    if blockContent['name'] == pluginname:
+                        if ('version' in blockContent['info']) and (blockContent['info']['version'] == version) and (not version is None):
+                            mostRecentTimestamp = blockDatetime
+                            mostRecentVersionBlock = block.getHash()
+                            break
+                        elif mostRecentTimestamp is None:
+                            mostRecentTimestamp = blockDatetime
+                            mostRecentVersionBlock = block.getHash()
+                        elif blockDatetime > mostRecentTimestamp:
+                            mostRecentTimestamp = blockDatetime
+                            mostRecentVersionBlock = block.getHash()
+                except Exception as e:
+                    pass
+
+            logger.warn('Only continue the installation is you are absolutely certain that you trust the plugin distributor. Public key of plugin distributor: %s' % publickey, timestamp = False)
+            installBlock(mostRecentVersionBlock)
+        else:
+            logger.error('Unknown data "%s"; must be public key or block hash.' % str(pkobh))
+            return
+    else:
+        logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin> [public key/block hash]')
+
+    return True
+
+def commandUninstallPlugin():
+    if len(sys.argv) >= 3:
+        uninstallPlugin(sys.argv[2])
+    else:
+        logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin>')
+
+    return True
+
+def commandSearchPlugin():
+    logger.info('This feature has not been created yet. Please check back later.')
+    return True
+
+def commandAddRepository():
+    if len(sys.argv) >= 3:
+        check()
+
+        blockhash = sys.argv[2]
+
+        if pluginapi.get_utils().validateHash(blockhash):
+            if Block.exists(blockhash):
+                try:
+                    blockContent = json.loads(Block(blockhash).getContent())
+
+                    pluginslist = dict()
+
+                    for pluginname, distributor in blockContent['plugins'].items():
+                        if pluginapi.get_utils().validatePubKey(distributor):
+                            pluginslist[pluginname] = distributor
+
+                    logger.debug('Found %s records in repository.' % len(pluginslist))
+
+                    if len(pluginslist) != 0:
+                        addRepository(blockhash, pluginslist)
+                        logger.info('Successfully added repository.')
+                    else:
+                        logger.error('Repository contains no records, not importing.')
+                except Exception as e:
+                    logger.error('Failed to parse block.', error = e)
+            else:
+                logger.error('Block hash not found. Perhaps it has not been synced yet?')
+                logger.debug('Is valid hash, but does not belong to a known block.')
+        else:
+            logger.error('Unknown data "%s"; must be block hash.' % str(pkobh))
+    else:
+        logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' [block hash]')
+
+    return True
+
+def commandRemoveRepository():
+    if len(sys.argv) >= 3:
+        check()
+
+        blockhash = sys.argv[2]
+
+        if pluginapi.get_utils().validateHash(blockhash):
+            if blockhash in getRepositories():
+                try:
+                    removeRepository(blockhash)
+                except Exception as e:
+                    logger.error('Failed to parse block.', error = e)
+            else:
+                logger.error('Repository has not been imported, nothing to remove.')
+        else:
+            logger.error('Unknown data "%s"; must be block hash.' % str(pkobh))
+    else:
+        logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' [block hash]')
+
+    return True
+
+def commandPublishPlugin():
+    if len(sys.argv) >= 3:
+        check()
+
+        pluginname = sanitize(sys.argv[2])
+        pluginfolder = pluginapi.plugins.get_folder(pluginname)
+
+        if os.path.exists(pluginfolder) and not os.path.isfile(pluginfolder):
+            block = pluginToBlock(pluginname)
+            logger.info('Plugin saved in block %s.' % block)
+        else:
+            logger.error('Plugin %s does not exist.' % pluginname, timestamp = False)
+    else:
+        logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin>')
+
+# event listeners
+
+def on_init(api, data = None):
+    global pluginapi
+    pluginapi = api
+    check()
+
+    # register some commands
+    api.commands.register(['install-plugin', 'installplugin', 'plugin-install', 'install', 'plugininstall'], commandInstallPlugin)
+    api.commands.register(['remove-plugin', 'removeplugin', 'plugin-remove', 'uninstall-plugin', 'uninstallplugin', 'plugin-uninstall', 'uninstall', 'remove', 'pluginremove'], commandUninstallPlugin)
+    api.commands.register(['search', 'filter-plugins', 'search-plugins', 'searchplugins', 'search-plugin', 'searchplugin', 'findplugin', 'find-plugin', 'filterplugin', 'plugin-search', 'pluginsearch'], commandSearchPlugin)
+    api.commands.register(['add-repo', 'add-repository', 'addrepo', 'addrepository', 'repository-add', 'repo-add', 'repoadd', 'addrepository', 'add-plugin-repository', 'add-plugin-repo', 'add-pluginrepo', 'add-pluginrepository', 'addpluginrepo', 'addpluginrepository'], commandAddRepository)
+    api.commands.register(['remove-repo', 'remove-repository', 'removerepo', 'removerepository', 'repository-remove', 'repo-remove', 'reporemove', 'removerepository', 'remove-plugin-repository', 'remove-plugin-repo', 'remove-pluginrepo', 'remove-pluginrepository', 'removepluginrepo', 'removepluginrepository', 'rm-repo', 'rm-repository', 'rmrepo', 'rmrepository', 'repository-rm', 'repo-rm', 'reporm', 'rmrepository', 'rm-plugin-repository', 'rm-plugin-repo', 'rm-pluginrepo', 'rm-pluginrepository', 'rmpluginrepo', 'rmpluginrepository'], commandRemoveRepository)
+    api.commands.register(['publish-plugin', 'plugin-publish', 'publishplugin', 'pluginpublish', 'publish'], commandPublishPlugin)
+
+    # add help menus once the features are actually implemented
+
+    return

onionr/static-data/default_config.json

@@ -12,5 +12,9 @@
             "output": true,
             "color": true
         }
+    },
+    "allocations":{
+        "disk": 1000000000,
+        "netTotal": 1000000000
     }
 }

onionr/static-data/default_plugin.py

@@ -1,11 +1,13 @@
 '''
-    Default plugin template file
+    $name plugin template file.
     Generated on $date by $user.
 '''
 
 # Imports some useful libraries
 import logger, config
 
+plugin_name = '$name'
+
 def on_init(api, data = None):
     '''
         This event is called after Onionr is initialized, but before the command

onionr/tests.py

@@ -14,7 +14,7 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import unittest, sys, os, base64, tarfile, shutil, simplecrypt, logger, btc
+import unittest, sys, os, base64, tarfile, shutil, simplecrypt, logger #, btc
 
 class OnionrTests(unittest.TestCase):
     def testPython3(self):
@@ -56,7 +56,7 @@ class OnionrTests(unittest.TestCase):
         myCore = core.Core()
         if not os.path.exists('data/peers.db'):
             myCore.createPeerDB()
-        if myCore.addPeer('6M5MXL237OK57ITHVYN5WGHANPGOMKS5C3PJLHBBNKFFJQOIDOJA====') and not myCore.addPeer('NFXHMYLMNFSAU==='):
+        if myCore.addPeer('6M5MXL237OK57ITHVYN5WGHANPGOMKS5C3PJLHBBNKFFJQOIDOJA====', '1cSix9Ao/yQSdo0sNif8cm2uTcYnSphb4JdZL/3WkN4=') and not myCore.addPeer('NFXHMYLMNFSAU===', '1cSix9Ao/yQSdo0sNif8cm2uTcYnSphb4JdZL/3WkN4='):
             self.assertTrue(True)
         else:
             self.assertTrue(False)