merge user-abstraction into webui work

.dockerignore

@@ -0,0 +1,4 @@
+onionr/data/**/*
+onionr/data
+RUN-WINDOWS.bat
+MY-RUN.sh

.gitignore

@@ -13,3 +13,4 @@ onionr/data-encrypted.dat
 onionr/.onionr-lock
 core
 .vscode/*
+venv/*

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "onionr/bitpeer"]
-	path = onionr/bitpeer
-	url = https://github.com/beardog108/bitpeer.py

Dockerfile

@@ -0,0 +1,28 @@
+FROM ubuntu:bionic
+
+#Base settings
+ENV HOME /root
+
+#Install needed packages
+RUN apt update && apt install -y python3 python3-dev python3-pip tor locales nano sqlite3
+
+RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
+    locale-gen
+ENV LANG en_US.UTF-8  
+ENV LANGUAGE en_US:en  
+ENV LC_ALL en_US.UTF-8  
+
+WORKDIR /srv/
+ADD ./requirements.txt /srv/requirements.txt
+RUN pip3 install -r requirements.txt
+
+WORKDIR /root/
+#Add Onionr source
+COPY . /root/
+VOLUME /root/data/
+
+#Set upstart command
+CMD bash
+
+#Expose ports
+EXPOSE 8080

Makefile

@@ -1,3 +1,5 @@
+PREFIX = /usr/local
+
 .DEFAULT_GOAL := setup
 
 setup:
@@ -5,16 +7,15 @@ setup:
 	-@cd onionr/static-data/ui/; ./compile.py
 
 install:
-	sudo rm -rf /usr/share/onionr/
+	cp -rfp ./onionr $(DESTDIR)$(PREFIX)/share/onionr
-	sudo rm -f /usr/bin/onionr
+	echo '#!/bin/sh' > $(DESTDIR)$(PREFIX)/bin/onionr
-	sudo cp -rp ./onionr /usr/share/onionr
+	echo 'cd $(DESTDIR)$(PREFIX)/share/onionr' > $(DESTDIR)$(PREFIX)/bin/onionr
-	sudo sh -c "echo \"#!/bin/sh\ncd /usr/share/onionr/\n./onionr.py \\\"\\\$$@\\\"\" > /usr/bin/onionr"
+	echo './onionr "$$@"' > $(DESTDIR)$(PREFIX)/bin/onionr
-	sudo chmod +x /usr/bin/onionr
+	chmod +x $(DESTDIR)$(PREFIX)/bin/onionr
-	sudo chown -R `whoami` /usr/share/onionr/
 
 uninstall:
-	sudo rm -rf /usr/share/onionr
+	rm -rf $(DESTDIR)$(PREFIX)/share/onionr
-	sudo rm -f /usr/bin/onionr
+	rm -f $(DESTDIR)$(PREFIX)/bin/onionr
 
 test:
 	@./RUN-LINUX.sh stop
@@ -27,7 +28,7 @@ test:
 
 soft-reset:
 	@echo "Soft-resetting Onionr..."
-	rm -f onionr/data/blocks/*.dat onionr/data/*.db | true > /dev/null 2>&1
+	rm -f onionr/data/blocks/*.dat onionr/data/*.db onionr/data/block-nonces.dat | true > /dev/null 2>&1
 	@./RUN-LINUX.sh version | grep -v "Failed" --color=always
 
 reset:

docs/whitepaper.md

@@ -86,6 +86,12 @@ Blocks are stored indefinitely until the allocated space is filled, at which poi
 
 ## Block Timestamping
 
-Onionr can provide evidence when a block was inserted by requesting other users to sign a hash of the current time with the block data hash: sha3_256(time + sha3_256(block data)).
+Onionr can provide evidence of when a block was inserted by requesting other users to sign a hash of the current time with the block data hash: sha3_256(time + sha3_256(block data)).
 
-This can be done either by the creator of the block prior to generation, or by any node after insertion.
+This can be done either by the creator of the block prior to generation, or by any node after insertion.
+
+In addition, randomness beacons such as the one operated by [NIST](https://beacon.nist.gov/home) or the hash of the latest blocks in a cryptocurrency network could be used to affirm that a block was at least not *created* before a given time.
+
+# Direct Connections
+
+We propose a system to 

onionr/api.py

@@ -20,11 +20,11 @@
 import flask
 from flask import request, Response, abort, send_from_directory
 from multiprocessing import Process
-from gevent.wsgi import WSGIServer
+from gevent.pywsgi import WSGIServer
 import sys, random, threading, hmac, hashlib, base64, time, math, os, json
 from core import Core
 from onionrblockapi import Block
-import onionrutils, onionrcrypto, blockimporter, onionrevents as events, logger, config
+import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config
 
 class API:
     '''
@@ -119,9 +119,7 @@ class API:
             '''
                 Simply define the request as not having yet failed, before every request.
             '''
-
             self.requestFailed = False
-
             return
 
         @app.after_request
@@ -241,16 +239,6 @@ class API:
                 resp = Response('Goodbye')
             elif action == 'ping':
                 resp = Response('pong')
-            elif action == 'site':
-                block = data
-                siteData = self._core.getData(data)
-                response = 'not found'
-                if siteData != '' and siteData != False:
-                    self.mimeType = 'text/html'
-                    response = siteData.split(b'-', 2)[-1]
-                resp = Response(response)
-            elif action == 'info':
-                resp = Response(json.dumps({'pubkey' : self._core._crypto.pubKey, 'host' : self._core.hsAdder}))
             elif action == "insertBlock":
                 response = {'success' : False, 'reason' : 'An unknown error occurred'}
 
@@ -399,13 +387,57 @@ class API:
                 pass
             else:
                 if sys.getsizeof(data) < 100000000:
-                    if blockimporter.importBlockFromData(data, self._core):
+                    try:
-                        resp = 'success'
+                        if blockimporter.importBlockFromData(data, self._core):
-                    else:
+                            resp = 'success'
-                        logger.warn('Error encountered importing uploaded block')
+                        else:
+                            logger.warn('Error encountered importing uploaded block')
+                    except onionrexceptions.BlacklistedBlock:
+                        logger.debug('uploaded block is blacklisted')
+                        pass
 
             resp = Response(resp)
             return resp
+
+        @app.route('/public/announce/', methods=['POST'])
+        def acceptAnnounce():
+            self.validateHost('public')
+            resp = 'failure'
+            powHash = ''
+            randomData = ''
+            newNode = ''
+            ourAdder = self._core.hsAddress.encode()
+            try:
+                newNode = request.form['node'].encode()
+            except KeyError:
+                logger.warn('No block specified for upload')
+                pass
+            else:
+                try:
+                    randomData = request.form['random']
+                    randomData = base64.b64decode(randomData)
+                except KeyError:
+                    logger.warn('No random data specified for upload')
+                else:
+                    nodes = newNode + self._core.hsAddress.encode()
+                    nodes = self._core._crypto.blake2bHash(nodes)
+                    powHash = self._core._crypto.blake2bHash(randomData + nodes)
+                    try:
+                        powHash = powHash.decode()
+                    except AttributeError:
+                        pass
+                    if powHash.startswith('0000'):
+                        try:
+                            newNode = newNode.decode()
+                        except AttributeError:
+                            pass
+                        if self._core.addAddress(newNode):
+                            resp = 'Success'
+                    else:
+                        logger.warn(newNode.decode() + ' failed to meet POW: ' + powHash)
+            resp = Response(resp)
+            return resp   
+
         @app.route('/public/')
         def public_handler():
             # Public means it is publicly network accessible
@@ -430,20 +462,11 @@ class API:
                 resp = Response(self._utils.getBlockDBHash())
             elif action == 'getBlockHashes':
                 resp = Response('\n'.join(self._core.getBlockList()))
-            elif action == 'announce':
-                if data != '':
-                    # TODO: require POW for this
-                    if self._core.addAddress(data):
-                        resp = Response('Success')
-                    else:
-                        resp = Response('')
-                else:
-                    resp = Response('')
             # setData should be something the communicator initiates, not this api
             elif action == 'getData':
                 resp = ''
                 if self._utils.validateHash(data):
-                    if not os.path.exists('data/blocks/' + data + '.db'):
+                    if os.path.exists('data/blocks/' + data + '.dat'):
                         block = Block(hash=data.encode(), core=self._core)
                         resp = base64.b64encode(block.getRaw().encode()).decode()
                 if len(resp) == 0:
@@ -477,7 +500,6 @@ class API:
         def authFail(err):
             self.requestFailed = True
             resp = Response("403")
-
             return resp
 
         @app.errorhandler(401)
@@ -490,11 +512,13 @@ class API:
             logger.info('Starting client on ' + self.host + ':' + str(bindPort) + '...', timestamp=False)
 
         try:
+            while len(self._core.hsAddress) == 0:
+                self._core.refreshFirstStartVars()
+                time.sleep(0.5)
             self.http_server = WSGIServer((self.host, bindPort), app)
             self.http_server.serve_forever()
         except KeyboardInterrupt:
             pass
-            #app.run(host=self.host, port=bindPort, debug=False, threaded=True)
         except Exception as e:
             logger.error(str(e))
             logger.fatal('Failed to start client on ' + self.host + ':' + str(bindPort) + ', exiting...')

onionr/blockimporter.py

@@ -20,6 +20,12 @@
 import core, onionrexceptions, logger
 def importBlockFromData(content, coreInst):
     retData = False
+
+    dataHash = coreInst._crypto.sha3Hash(content)
+
+    if coreInst._blacklist.inBlacklist(dataHash):
+        raise onionrexceptions.BlacklistedBlock('%s is a blacklisted block' % (dataHash,))
+
     if not isinstance(coreInst, core.Core):
         raise Exception("coreInst must be an Onionr core instance")
 
@@ -30,11 +36,15 @@ def importBlockFromData(content, coreInst):
 
     metas = coreInst._utils.getBlockMetadataFromData(content) # returns tuple(metadata, meta), meta is also in metadata
     metadata = metas[0]
-    if coreInst._utils.validateMetadata(metadata): # check if metadata is valid
+    if coreInst._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid
         if coreInst._crypto.verifyPow(content): # check if POW is enough/correct
             logger.info('Block passed proof, saving.')
-            blockHash = coreInst.setData(content)
+            try:
-            blockHash = coreInst.addToBlockDB(blockHash, dataSaved=True)
+                blockHash = coreInst.setData(content)
-            coreInst._utils.processBlockMetadata(blockHash) # caches block metadata values to block database
+            except onionrexceptions.DiskAllocationReached:
-            retData = True
+                pass
+            else:
+                coreInst.addToBlockDB(blockHash, dataSaved=True)
+                coreInst._utils.processBlockMetadata(blockHash) # caches block metadata values to block database
+                retData = True
     return retData

onionr/communicator2.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 '''
-    Onionr - P2P Microblogging Platform & Social network.
+    Onionr - P2P Anonymous Storage Network
 
     This file contains both the OnionrCommunicate class for communcating with peers
     and code to operate as a daemon, getting commands from the command queue database (see core.Core.daemonQueue)
@@ -21,11 +21,14 @@
 '''
 import sys, os, core, config, json, requests, time, logger, threading, base64, onionr
 import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block
+import onionrdaemontools
 from defusedxml import minidom
 
 class OnionrCommunicatorDaemon:
     def __init__(self, debug, developmentMode):
 
+        self.isOnline = True # Assume we're connected to the internet
+
         # list of timer instances
         self.timers = []
 
@@ -48,6 +51,8 @@ class OnionrCommunicatorDaemon:
         # lists of connected peers and peers we know we can't reach currently
         self.onlinePeers = []
         self.offlinePeers = []
+        self.cooldownPeer = {}
+        self.connectTimes = {}
         self.peerProfiles = [] # list of peer's profiles (onionrpeers.PeerProfile instances)
 
         # amount of threads running by name, used to prevent too many
@@ -69,28 +74,34 @@ class OnionrCommunicatorDaemon:
         # Loads in and starts the enabled plugins
         plugins.reload()
 
+        # daemon tools are misc daemon functions, e.g. announce to online peers
+        # intended only for use by OnionrCommunicatorDaemon
+        #self.daemonTools = onionrdaemontools.DaemonTools(self)
+        self.daemonTools = onionrdaemontools.DaemonTools(self)
+
         if debug or developmentMode:
             OnionrCommunicatorTimers(self, self.heartbeat, 10)
 
-        # Print nice header thing :)
-        if config.get('general.display_header', True) and not self.shutdown:
-            self.header()
-
         # Set timers, function reference, seconds
         # requiresPeer True means the timer function won't fire if we have no connected peers
         OnionrCommunicatorTimers(self, self.daemonCommands, 5)
         OnionrCommunicatorTimers(self, self.detectAPICrash, 5)
-        peerPoolTimer = OnionrCommunicatorTimers(self, self.getOnlinePeers, 60)
+        peerPoolTimer = OnionrCommunicatorTimers(self, self.getOnlinePeers, 60, maxThreads=1)
-        OnionrCommunicatorTimers(self, self.lookupBlocks, 7, requiresPeer=True, maxThreads=1)
+        OnionrCommunicatorTimers(self, self.lookupBlocks, self._core.config.get('timers.lookupBlocks'), requiresPeer=True, maxThreads=1)
-        OnionrCommunicatorTimers(self, self.getBlocks, 10, requiresPeer=True)
+        OnionrCommunicatorTimers(self, self.getBlocks, self._core.config.get('timers.getBlocks'), requiresPeer=True)
         OnionrCommunicatorTimers(self, self.clearOfflinePeer, 58)
+        OnionrCommunicatorTimers(self, self.daemonTools.cleanOldBlocks, 65)
         OnionrCommunicatorTimers(self, self.lookupKeys, 60, requiresPeer=True)
         OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True)
+        OnionrCommunicatorTimers(self, self.daemonTools.cooldownPeer, 30, requiresPeer=True)
+        netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600)
+        announceTimer = OnionrCommunicatorTimers(self, self.daemonTools.announceNode, 305, requiresPeer=True, maxThreads=1)
         cleanupTimer = OnionrCommunicatorTimers(self, self.peerCleanup, 300, requiresPeer=True)
 
         # set loop to execute instantly to load up peer pool (replaced old pool init wait)
         peerPoolTimer.count = (peerPoolTimer.frequency - 1)
         cleanupTimer.count = (cleanupTimer.frequency - 60)
+        announceTimer.count = (cleanupTimer.frequency - 60)
 
         # Main daemon loop, mainly for calling timers, don't do any complex operations here to avoid locking
         try:
@@ -105,14 +116,14 @@ class OnionrCommunicatorDaemon:
             pass
 
         logger.info('Goodbye.')
-        self._core._utils.localCommand('shutdown')
+        self._core._utils.localCommand('shutdown') # shutdown the api
         time.sleep(0.5)
 
     def lookupKeys(self):
         '''Lookup new keys'''
         logger.debug('Looking up new keys...')
         tryAmount = 1
-        for i in range(tryAmount):
+        for i in range(tryAmount): # amount of times to ask peers for new keys
             # Download new key list from random online peers
             peer = self.pickOnlinePeer()
             newKeys = self.peerAction(peer, action='kex')
@@ -139,6 +150,12 @@ class OnionrCommunicatorDaemon:
         existingBlocks = self._core.getBlockList()
         triedPeers = [] # list of peers we've tried this time around
         for i in range(tryAmount):
+            # check if disk allocation is used
+            if not self.isOnline:
+                break
+            if self._core._utils.storageCounter.isFull():
+                logger.debug('Not looking up new blocks due to maximum amount of allowed disk space used')
+                break
             peer = self.pickOnlinePeer() # select random online peer
             # if we've already tried all the online peers this time around, stop
             if peer in triedPeers:
@@ -153,7 +170,7 @@ class OnionrCommunicatorDaemon:
             if newDBHash != self._core.getAddressInfo(peer, 'DBHash'):
                 self._core.setAddressInfo(peer, 'DBHash', newDBHash)
                 try:
-                    newBlocks = self.peerAction(peer, 'getBlockHashes')
+                    newBlocks = self.peerAction(peer, 'getBlockHashes') # get list of new block hashes
                 except Exception as error:
                     logger.warn("could not get new blocks with " + peer, error=error)
                     newBlocks = False
@@ -164,20 +181,31 @@ class OnionrCommunicatorDaemon:
                             # if newline seperated string is valid hash
                             if not i in existingBlocks:
                                 # if block does not exist on disk and is not already in block queue
-                                if i not in self.blockQueue:
+                                if i not in self.blockQueue and not self._core._blacklist.inBlacklist(i):
-                                    self.blockQueue.append(i)
+                                    self.blockQueue.append(i) # add blocks to download queue
         self.decrementThreadCount('lookupBlocks')
         return
 
     def getBlocks(self):
         '''download new blocks in queue'''
         for blockHash in self.blockQueue:
-            if self.shutdown:
+            removeFromQueue = True
+            if self.shutdown or not self.isOnline:
+                # Exit loop if shutting down or offline
                 break
+            # Do not download blocks being downloaded or that are already saved (edge cases)
             if blockHash in self.currentDownloading:
                 logger.debug('ALREADY DOWNLOADING ' + blockHash)
                 continue
-            self.currentDownloading.append(blockHash)
+            if blockHash in self._core.getBlockList():
+                logger.debug('%s is already saved' % (blockHash,))
+                self.blockQueue.remove(blockHash)
+                continue
+            if self._core._blacklist.inBlacklist(blockHash):
+                continue
+            if self._core._utils.storageCounter.isFull():
+                break
+            self.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block
             logger.info("Attempting to download %s..." % blockHash)
             peerUsed = self.pickOnlinePeer()
             content = self.peerAction(peerUsed, 'getData', data=blockHash) # block content from random peer (includes metadata)
@@ -197,16 +225,25 @@ class OnionrCommunicatorDaemon:
                     metas = self._core._utils.getBlockMetadataFromData(content) # returns tuple(metadata, meta), meta is also in metadata
                     metadata = metas[0]
                     #meta = metas[1]
-                    if self._core._utils.validateMetadata(metadata): # check if metadata is valid
+                    if self._core._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid, and verify nonce
                         if self._core._crypto.verifyPow(content): # check if POW is enough/correct
-                            logger.info('Block passed proof, saving.')
+                            logger.info('Block passed proof, attempting save.')
-                            self._core.setData(content)
+                            try:
-                            self._core.addToBlockDB(blockHash, dataSaved=True)
+                                self._core.setData(content)
-                            self._core._utils.processBlockMetadata(blockHash) # caches block metadata values to block database
+                            except onionrexceptions.DiskAllocationReached:
+                                logger.error("Reached disk allocation allowance, cannot save this block.")
+                                removeFromQueue = False
+                            else:
+                                self._core.addToBlockDB(blockHash, dataSaved=True)
+                                self._core._utils.processBlockMetadata(blockHash) # caches block metadata values to block database
                         else:
                             logger.warn('POW failed for block ' + blockHash)
                     else:
-                        logger.warn('Metadata for ' + blockHash + ' is invalid.')
+                        if self._core._blacklist.inBlacklist(realHash):
+                            logger.warn('%s is blacklisted' % (realHash,))
+                        else:
+                            logger.warn('Metadata for ' + blockHash + ' is invalid.')
+                            self._core._blacklist.addToDB(blockHash)
                 else:
                     # if block didn't meet expected hash
                     tempHash = self._core._crypto.sha3Hash(content) # lazy hack, TODO use var
@@ -217,7 +254,8 @@ class OnionrCommunicatorDaemon:
                     # Punish peer for sharing invalid block (not always malicious, but is bad regardless)
                     onionrpeers.PeerProfiles(peerUsed, self._core).addScore(-50)  
                     logger.warn('Block hash validation failed for ' + blockHash + ' got ' + tempHash)
-                self.blockQueue.remove(blockHash) # remove from block queue both if success or false
+                if removeFromQueue:
+                    self.blockQueue.remove(blockHash) # remove from block queue both if success or false
             self.currentDownloading.remove(blockHash)
         self.decrementThreadCount('getBlocks')
         return
@@ -260,7 +298,7 @@ class OnionrCommunicatorDaemon:
         '''Manages the self.onlinePeers attribute list, connects to more peers if we have none connected'''
 
         logger.info('Refreshing peer pool.')
-        maxPeers = 6
+        maxPeers = int(config.get('peers.maxConnect'))
         needed = maxPeers - len(self.onlinePeers)
 
         for i in range(needed):
@@ -278,8 +316,9 @@ class OnionrCommunicatorDaemon:
     def addBootstrapListToPeerList(self, peerList):
         '''Add the bootstrap list to the peer list (no duplicates)'''
         for i in self._core.bootstrapList:
-            if i not in peerList and i not in self.offlinePeers and i != self._core.hsAdder:
+            if i not in peerList and i not in self.offlinePeers and i != self._core.hsAddress:
                 peerList.append(i)
+                self._core.addAddress(i)
 
     def connectNewPeer(self, peer='', useBootstrap=False):
         '''Adds a new random online peer to self.onlinePeers'''
@@ -300,7 +339,9 @@ class OnionrCommunicatorDaemon:
             self.addBootstrapListToPeerList(peerList)
 
         for address in peerList:
-            if len(address) == 0 or address in tried or address in self.onlinePeers:
+            if not config.get('tor.v3onions') and len(address) == 62:
+                continue
+            if len(address) == 0 or address in tried or address in self.onlinePeers or address in self.cooldownPeer:
                 continue
             if self.shutdown:
                 return
@@ -309,6 +350,7 @@ class OnionrCommunicatorDaemon:
                 time.sleep(0.1)
                 if address not in self.onlinePeers:
                     self.onlinePeers.append(address)
+                    self.connectTimes[address] = self._core._utils.getEpoch()
                 retData = address
                 
                 # add peer to profile list if they're not in it
@@ -323,6 +365,17 @@ class OnionrCommunicatorDaemon:
                 logger.debug('Failed to connect to ' + address)
         return retData
 
+    def removeOnlinePeer(self, peer):
+        '''Remove an online peer'''
+        try:
+            del self.connectTimes[peer]
+        except KeyError:
+            pass
+        try:
+            self.onlinePeers.remove(peer)
+        except ValueError:
+            pass
+
     def peerCleanup(self):
         '''This just calls onionrpeers.cleanupPeers, which removes dead or bad peers (offline too long, too slow)'''
         onionrpeers.peerCleanup(self._core)
@@ -354,8 +407,9 @@ class OnionrCommunicatorDaemon:
         if retData == False:
             try:
                 self.getPeerProfileInstance(peer).addScore(-10)
-                self.onlinePeers.remove(peer)
+                self.removeOnlinePeer(peer)
-                self.getOnlinePeers() # Will only add a new peer to pool if needed
+                if action != 'ping':
+                    self.getOnlinePeers() # Will only add a new peer to pool if needed
             except ValueError:
                 pass
         else:
@@ -437,17 +491,10 @@ class OnionrCommunicatorDaemon:
 
     def announce(self, peer):
         '''Announce to peers our address'''
-        announceCount = 0
+        if self.daemonTools.announceNode():
-        announceAmount = 2
+            logger.info('Successfully introduced node to ' + peer)
-        for peer in self.onlinePeers:
+        else:
-            announceCount += 1
+            logger.warn('Could not introduce node.')
-            if self.peerAction(peer, 'announce', self._core.hsAdder) == 'Success':
-                logger.info('Successfully introduced node to ' + peer)
-                break
-            else:
-                if announceCount == announceAmount:
-                    logger.warn('Could not introduce node. Try again soon')
-                    break
 
     def detectAPICrash(self):
         '''exit if the api server crashes/stops'''
@@ -463,13 +510,6 @@ class OnionrCommunicatorDaemon:
                 self.shutdown = True
         self.decrementThreadCount('detectAPICrash')
 
-    def header(self, message = logger.colors.fg.pink + logger.colors.bold + 'Onionr' + logger.colors.reset + logger.colors.fg.pink + ' has started.'):
-        if os.path.exists('static-data/header.txt'):
-            with open('static-data/header.txt', 'rb') as file:
-                # only to stdout, not file or log or anything
-                sys.stderr.write(file.read().decode().replace('P', logger.colors.fg.pink).replace('W', logger.colors.reset + logger.colors.bold).replace('G', logger.colors.fg.green).replace('\n', logger.colors.reset + '\n').replace('B', logger.colors.bold).replace('V', onionr.ONIONR_VERSION))
-                logger.info(logger.colors.fg.lightgreen + '-> ' + str(message) + logger.colors.reset + logger.colors.fg.lightgreen + ' <-\n')
-
 class OnionrCommunicatorTimers:
     def __init__(self, daemonInstance, timerFunction, frequency, makeThread=True, threadAmount=1, maxThreads=5, requiresPeer=False):
         self.timerFunction = timerFunction

onionr/core.py

@@ -1,5 +1,5 @@
 '''
-    Onionr - P2P Microblogging Platform & Social network
+    Onionr - P2P Anonymous Storage Network
 
     Core Onionr library, useful for external programs. Handles peer & data processing
 '''
@@ -21,7 +21,8 @@ import sqlite3, os, sys, time, math, base64, tarfile, getpass, simplecrypt, hash
 from onionrblockapi import Block
 
 import onionrutils, onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions, onionrvalues
-
+import onionrblacklist
+import dbcreator
 if sys.version_info < (3, 6):
     try:
         import sha3
@@ -40,13 +41,18 @@ class Core:
             self.blockDB = 'data/blocks.db'
             self.blockDataLocation = 'data/blocks/'
             self.addressDB = 'data/address.db'
-            self.hsAdder = ''
+            self.hsAddress = ''
             self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt'
             self.bootstrapList = []
             self.requirements = onionrvalues.OnionrValues()
             self.torPort = torPort
+            self.dataNonceFile = 'data/block-nonces.dat'
+            self.dbCreate = dbcreator.DBCreator(self)
 
             self.usageFile = 'data/disk-usage.txt'
+            self.config = config
+
+            self.maxBlockSize = 10000000 # max block size in bytes
 
             if not os.path.exists('data/'):
                 os.mkdir('data/')
@@ -57,7 +63,7 @@ class Core:
 
             if os.path.exists('data/hs/hostname'):
                 with open('data/hs/hostname', 'r') as hs:
-                    self.hsAdder = hs.read().strip()
+                    self.hsAddress = hs.read().strip()
 
             # Load bootstrap address list
             if os.path.exists(self.bootstrapFileLocation):
@@ -71,6 +77,7 @@ class Core:
             self._utils = onionrutils.OnionrUtils(self)
             # Initialize the crypto object
             self._crypto = onionrcrypto.OnionrCrypto(self)
+            self._blacklist = onionrblacklist.OnionrBlackList(self)
 
         except Exception as error:
             logger.error('Failed to initialize core Onionr library.', error=error)
@@ -78,6 +85,12 @@ class Core:
             sys.exit(1)
         return
 
+    def refreshFirstStartVars(self):
+        '''Hack to refresh some vars which may not be set on first start'''
+        if os.path.exists('data/hs/hostname'):
+            with open('data/hs/hostname', 'r') as hs:
+                self.hsAddress = hs.read().strip()
+
     def addPeer(self, peerID, powID, name=''):
         '''
             Adds a public key to the key database (misleading function name)
@@ -92,7 +105,7 @@ class Core:
         conn = sqlite3.connect(self.peerDB)
         hashID = self._crypto.pubKeyHashID(peerID)
         c = conn.cursor()
-        t = (peerID, name, 'unknown', hashID, powID)
+        t = (peerID, name, 'unknown', hashID, powID, 0)
 
         for i in c.execute("SELECT * FROM PEERS where id = '" + peerID + "';"):
             try:
@@ -103,7 +116,7 @@ class Core:
                 pass
             except IndexError:
                 pass
-        c.execute('INSERT INTO peers (id, name, dateSeen, pow, hashID) VALUES(?, ?, ?, ?, ?);', t)
+        c.execute('INSERT INTO peers (id, name, dateSeen, pow, hashID, trust) VALUES(?, ?, ?, ?, ?, ?);', t)
         conn.commit()
         conn.close()
 
@@ -125,7 +138,6 @@ class Core:
             for i in c.execute("SELECT * FROM adders where address = '" + address + "';"):
                 try:
                     if i[0] == address:
-                        logger.warn('Not adding existing address')
                         conn.close()
                         return False
                 except ValueError:
@@ -158,14 +170,15 @@ class Core:
             conn.close()
 
             events.event('address_remove', data = {'address': address}, onionr = None)
-
             return True
         else:
             return False
 
     def removeBlock(self, block):
         '''
-            remove a block from this node
+            remove a block from this node (does not automatically blacklist)
+
+            **You may want blacklist.addToDB(blockHash)
         '''
         if self._utils.validateHash(block):
             conn = sqlite3.connect(self.blockDB)
@@ -174,97 +187,36 @@ class Core:
             c.execute('Delete from hashes where hash=?;', t)
             conn.commit()
             conn.close()
+            blockFile = 'data/blocks/' + block + '.dat'
+            dataSize = 0
             try:
-                os.remove('data/blocks/' + block + '.dat')
+                ''' Get size of data when loaded as an object/var, rather than on disk, 
+                    to avoid conflict with getsizeof when saving blocks
+                '''
+                with open(blockFile, 'r') as data:
+                    dataSize = sys.getsizeof(data.read())
+                self._utils.storageCounter.removeBytes(dataSize)
+                os.remove(blockFile)
             except FileNotFoundError:
                 pass
 
     def createAddressDB(self):
         '''
             Generate the address database
-
-            types:
-                1: I2P b32 address
-                2: Tor v2 (like facebookcorewwwi.onion)
-                3: Tor v3
         '''
-        conn = sqlite3.connect(self.addressDB)
+        self.dbCreate.createAddressDB()
-        c = conn.cursor()
-        c.execute('''CREATE TABLE adders(
-            address text,
-            type int,
-            knownPeer text,
-            speed int,
-            success int,
-            DBHash text,
-            powValue text,
-            failure int,
-            lastConnect int,
-            lastConnectAttempt int,
-            trust int
-            );
-        ''')
-        conn.commit()
-        conn.close()
 
     def createPeerDB(self):
         '''
             Generate the peer sqlite3 database and populate it with the peers table.
         '''
-        # generate the peer database
+        self.dbCreate.createPeerDB()
-        conn = sqlite3.connect(self.peerDB)
-        c = conn.cursor()
-        c.execute('''CREATE TABLE peers(
-            ID text not null,
-            name text,
-            adders text,
-            blockDBHash text,
-            forwardKey text,
-            dateSeen not null,
-            bytesStored int,
-            trust int,
-            pubkeyExchanged int,
-            hashID text,
-            pow text not null);
-        ''')
-        conn.commit()
-        conn.close()
-        return
 
     def createBlockDB(self):
         '''
             Create a database for blocks
-
-            hash         - the hash of a block
-            dateReceived - the date the block was recieved, not necessarily when it was created
-            decrypted    - if we can successfully decrypt the block (does not describe its current state)
-            dataType     - data type of the block
-            dataFound    - if the data has been found for the block
-            dataSaved    - if the data has been saved for the block
-            sig    - optional signature by the author (not optional if author is specified)
-            author       - multi-round partial sha3-256 hash of authors public key
-            dateClaimed  - timestamp claimed inside the block, only as trustworthy as the block author is
         '''
-        if os.path.exists(self.blockDB):
+        self.dbCreate.createBlockDB()
-            raise Exception("Block database already exists")
-        conn = sqlite3.connect(self.blockDB)
-        c = conn.cursor()
-        c.execute('''CREATE TABLE hashes(
-            hash text not null,
-            dateReceived int,
-            decrypted int,
-            dataType text,
-            dataFound int,
-            dataSaved int,
-            sig text,
-            author text,
-            dateClaimed int
-            );
-        ''')
-        conn.commit()
-        conn.close()
-
-        return
 
     def addToBlockDB(self, newHash, selfInsert=False, dataSaved=False):
         '''
@@ -304,16 +256,26 @@ class Core:
 
         return data
 
-    def setData(self, data):
+    def _getSha3Hash(self, data):
-        '''
-            Set the data assciated with a hash
-        '''
-        data = data
         hasher = hashlib.sha3_256()
         if not type(data) is bytes:
             data = data.encode()
         hasher.update(data)
         dataHash = hasher.hexdigest()
+        return dataHash
+
+    def setData(self, data):
+        '''
+            Set the data assciated with a hash
+        '''
+        data = data
+        dataSize = sys.getsizeof(data)
+
+        if not type(data) is bytes:
+            data = data.encode()
+            
+        dataHash = self._getSha3Hash(data)
+
         if type(dataHash) is bytes:
             dataHash = dataHash.decode()
         blockFileName = self.blockDataLocation + dataHash + '.dat'
@@ -321,15 +283,19 @@ class Core:
             pass # TODO: properly check if block is already saved elsewhere
             #raise Exception("Data is already set for " + dataHash)
         else:
-            blockFile = open(blockFileName, 'wb')
+            if self._utils.storageCounter.addBytes(dataSize) != False:
-            blockFile.write(data)
+                blockFile = open(blockFileName, 'wb')
-            blockFile.close()
+                blockFile.write(data)
-
+                blockFile.close()
-        conn = sqlite3.connect(self.blockDB)
+                conn = sqlite3.connect(self.blockDB)
-        c = conn.cursor()
+                c = conn.cursor()
-        c.execute("UPDATE hashes SET dataSaved=1 WHERE hash = '" + dataHash + "';")
+                c.execute("UPDATE hashes SET dataSaved=1 WHERE hash = '" + dataHash + "';")
-        conn.commit()
+                conn.commit()
-        conn.close()
+                conn.close()
+                with open(self.dataNonceFile, 'a') as nonceFile:
+                    nonceFile.write(dataHash + '\n')
+            else:
+                raise onionrexceptions.DiskAllocationReached
 
         return dataHash
 
@@ -411,18 +377,22 @@ class Core:
         '''
             Add a command to the daemon queue, used by the communication daemon (communicator.py)
         '''
+        retData = True
         # Intended to be used by the web server
         date = self._utils.getEpoch()
         conn = sqlite3.connect(self.queueDB)
         c = conn.cursor()
         t = (command, data, date)
-        c.execute('INSERT INTO commands (command, data, date) VALUES(?, ?, ?)', t)
+        try:
-        conn.commit()
+            c.execute('INSERT INTO commands (command, data, date) VALUES(?, ?, ?)', t)
-        conn.close()
+            conn.commit()
-
+            conn.close()
+        except sqlite3.OperationalError:
+            retData = False
+            self.daemonQueue()
         events.event('queue_push', data = {'command': command, 'data': data}, onionr = None)
 
-        return
+        return retData
 
     def clearDaemonQueue(self):
         '''
@@ -456,19 +426,23 @@ class Core:
         conn.close()
         return addressList
 
-    def listPeers(self, randomOrder=True, getPow=False):
+    def listPeers(self, randomOrder=True, getPow=False, trust=0):
         '''
             Return a list of public keys (misleading function name)
 
             randomOrder determines if the list should be in a random order
+            trust sets the minimum trust to list
         '''
         conn = sqlite3.connect(self.peerDB)
         c = conn.cursor()
         payload = ""
+        if trust not in (0, 1, 2):
+            logger.error('Tried to select invalid trust.')
+            return
         if randomOrder:
-            payload = 'SELECT * FROM peers ORDER BY RANDOM();'
+            payload = 'SELECT * FROM peers where trust >= %s ORDER BY RANDOM();' % (trust,)
         else:
-            payload = 'SELECT * FROM peers;'
+            payload = 'SELECT * FROM peers where trust >= %s;' % (trust,)
         peerList = []
         for i in c.execute(payload):
             try:
@@ -592,7 +566,7 @@ class Core:
         if unsaved:
             execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();'
         else:
-            execute = 'SELECT hash FROM hashes ORDER BY dateReceived DESC;'
+            execute = 'SELECT hash FROM hashes ORDER BY dateReceived ASC;'
         rows = list()
         for row in c.execute(execute):
             for i in row:
@@ -677,6 +651,18 @@ class Core:
         '''
         retData = False
 
+        # check nonce
+        dataNonce = self._utils.bytesToStr(self._crypto.sha3Hash(data))
+        try:
+            with open(self.dataNonceFile, 'r') as nonces:
+                if dataNonce in nonces:
+                    return retData
+        except FileNotFoundError:
+            pass
+        # record nonce
+        with open(self.dataNonceFile, 'a') as nonceFile:
+            nonceFile.write(dataNonce + '\n')
+
         if meta is None:
             meta = dict()
 
@@ -688,6 +674,7 @@ class Core:
         signature = ''
         signer = ''
         metadata = {}
+        # metadata is full block metadata, meta is internal, user specified metadata
 
         # only use header if not set in provided meta
         if not header is None:
@@ -735,7 +722,7 @@ class Core:
         metadata['sig'] = signature
         metadata['signer'] = signer
         metadata['time'] = str(self._utils.getEpoch())
-
+    
         # send block data (and metadata) to POW module to get tokenized block data
         proof = onionrproofs.POW(metadata, data)
         payload = proof.waitForResult()

onionr/dbcreator.py

@@ -0,0 +1,108 @@
+'''
+    Onionr - P2P Anonymous Data Storage & Sharing
+
+    DBCreator, creates sqlite3 databases used by Onionr
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sqlite3, os
+class DBCreator:
+    def __init__(self, coreInst):
+        self.core = coreInst
+
+    def createAddressDB(self):
+        '''
+            Generate the address database
+
+            types:
+                1: I2P b32 address
+                2: Tor v2 (like facebookcorewwwi.onion)
+                3: Tor v3
+        '''
+        conn = sqlite3.connect(self.core.addressDB)
+        c = conn.cursor()
+        c.execute('''CREATE TABLE adders(
+            address text,
+            type int,
+            knownPeer text,
+            speed int,
+            success int,
+            DBHash text,
+            powValue text,
+            failure int,
+            lastConnect int,
+            lastConnectAttempt int,
+            trust int
+            );
+        ''')
+        conn.commit()
+        conn.close()
+
+    def createPeerDB(self):
+        '''
+            Generate the peer sqlite3 database and populate it with the peers table.
+        '''
+        # generate the peer database
+        conn = sqlite3.connect(self.core.peerDB)
+        c = conn.cursor()
+        c.execute('''CREATE TABLE peers(
+            ID text not null,
+            name text,
+            adders text,
+            forwardKey text,
+            dateSeen not null,
+            bytesStored int,
+            trust int,
+            pubkeyExchanged int,
+            hashID text,
+            pow text not null);
+        ''')
+        conn.commit()
+        conn.close()
+        return
+
+    def createBlockDB(self):
+        '''
+            Create a database for blocks
+
+            hash         - the hash of a block
+            dateReceived - the date the block was recieved, not necessarily when it was created
+            decrypted    - if we can successfully decrypt the block (does not describe its current state)
+            dataType     - data type of the block
+            dataFound    - if the data has been found for the block
+            dataSaved    - if the data has been saved for the block
+            sig    - optional signature by the author (not optional if author is specified)
+            author       - multi-round partial sha3-256 hash of authors public key
+            dateClaimed  - timestamp claimed inside the block, only as trustworthy as the block author is
+        '''
+        if os.path.exists(self.core.blockDB):
+            raise Exception("Block database already exists")
+        conn = sqlite3.connect(self.core.blockDB)
+        c = conn.cursor()
+        c.execute('''CREATE TABLE hashes(
+            hash text not null,
+            dateReceived int,
+            decrypted int,
+            dataType text,
+            dataFound int,
+            dataSaved int,
+            sig text,
+            author text,
+            dateClaimed int
+            );
+        ''')
+        conn.commit()
+        conn.close()
+        return

onionr/netcontroller.py

@@ -18,7 +18,7 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 
-import subprocess, os, random, sys, logger, time, signal
+import subprocess, os, random, sys, logger, time, signal, config
 from onionrblockapi import Block
 
 class NetController:
@@ -33,6 +33,7 @@ class NetController:
         self.hsPort = hsPort
         self._torInstnace = ''
         self.myID = ''
+        config.reload()
         '''
             if os.path.exists(self.torConfigLocation):
                 torrc = open(self.torConfigLocation, 'r')
@@ -47,11 +48,15 @@ class NetController:
         '''
             Generate a torrc file for our tor instance
         '''
-
+        hsVer = '# v2 onions'
+        if config.get('tor.v3onions'):
+            hsVer = 'HiddenServiceVersion 3'
+            logger.info('Using v3 onions :)')
         if os.path.exists(self.torConfigLocation):
             os.remove(self.torConfigLocation)
         torrcData = '''SocksPort ''' + str(self.socksPort) + '''
 HiddenServiceDir data/hs/
+\n''' + hsVer + '''\n
 HiddenServicePort 80 127.0.0.1:''' + str(self.hsPort) + '''
 DataDirectory data/tordata/
         '''
@@ -97,7 +102,7 @@ DataDirectory data/tordata/
                 elif 'Opening Socks listener' in line.decode():
                     logger.debug(line.decode().replace('\n', ''))
             else:
-                logger.fatal('Failed to start Tor. Try killing any other Tor processes owned by this user.')
+                logger.fatal('Failed to start Tor. Maybe a stray instance of Tor used by Onionr is still running?')
                 return False
         except KeyboardInterrupt:
             logger.fatal("Got keyboard interrupt.")

onionr/onionr.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 '''
-    Onionr - P2P Microblogging Platform & Social network.
+    Onionr - P2P Anonymous Storage Network
 
     Onionr is the name for both the protocol and the original/reference software.
 
@@ -32,7 +32,7 @@ import onionrutils
 from onionrutils import OnionrUtils
 from netcontroller import NetController
 from onionrblockapi import Block
-import onionrproofs
+import onionrproofs, onionrexceptions, onionrusers
 
 try:
     from urllib3.contrib.socks import SOCKSProxyManager
@@ -40,7 +40,7 @@ except ImportError:
     raise Exception("You need the PySocks module (for use with socks5 proxy to use Tor)")
 
 ONIONR_TAGLINE = 'Anonymous P2P Platform - GPLv3 - https://Onionr.VoidNet.Tech'
-ONIONR_VERSION = '0.1.1' # for debugging and stuff
+ONIONR_VERSION = '0.2.0' # for debugging and stuff
 ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION)
 API_VERSION = '4' # increments of 1; only change when something fundemental about how the API works changes. This way other nodes know how to communicate without learning too much information about you.
 
@@ -135,18 +135,18 @@ class Onionr:
             self.onionrCore.createAddressDB()
 
         # Get configuration
-
+        if type(config.get('client.hmac')) is type(None):
-        if not data_exists:
+            config.set('client.hmac', base64.b16encode(os.urandom(32)).decode('utf-8'), savefile=True)
-            # Generate default config
+        if type(config.get('client.port')) is type(None):
-            # Hostname should only be set if different from 127.x.x.x. Important for DNS rebinding attack prevention.
+            randomPort = 0
-            if self.debug:
+            while randomPort < 1024:
-                randomPort = 8080
+                randomPort = self.onionrCore._crypto.secrets.randbelow(65535)
-            else:
+            config.set('client.port', randomPort, savefile=True)
-                while True:
+        if type(config.get('client.participate')) is type(None):
-                    randomPort = random.randint(1024, 65535)
+            config.set('client.participate', True, savefile=True)
-                    if self.onionrUtils.checkPort(randomPort):
+        if type(config.get('client.api_version')) is type(None):
-                        break
+            config.set('client.api_version', API_VERSION, savefile=True)
-            config.set('client', {'participate': True, 'hmac': base64.b16encode(os.urandom(32)).decode('utf-8'), 'port': randomPort, 'api_version': API_VERSION}, True)
+    
 
         self.cmds = {
             '': self.showHelpSuggestion,
@@ -186,6 +186,8 @@ class Onionr:
             'addaddress': self.addAddress,
             'list-peers': self.listPeers,
 
+            'blacklist-block': self.banBlock,
+
             'add-file': self.addFile,
             'addfile': self.addFile,
             'listconn': self.listConn,
@@ -208,7 +210,9 @@ class Onionr:
             'getpass': self.printWebPassword,
             'get-pass': self.printWebPassword,
             'getpasswd': self.printWebPassword,
-            'get-passwd': self.printWebPassword
+            'get-passwd': self.printWebPassword,
+
+            'friend': self.friendCmd
         }
 
         self.cmdhelp = {
@@ -230,7 +234,9 @@ class Onionr:
             'listconn': 'list connected peers',
             'kex': 'exchange keys with peers (done automatically)',
             'pex': 'exchange addresses with peers (done automatically)',
+            'blacklist-block': 'deletes a block by hash and permanently removes it from your node',
             'introduce': 'Introduce your node to the public Onionr network',
+            'friend': '[add|remove] [public key/id]'
         }
 
         # initialize plugins
@@ -257,6 +263,68 @@ class Onionr:
 
     def getCommands(self):
         return self.cmds
+    
+    def friendCmd(self):
+        '''List, add, or remove friend(s)
+        Changes their peer DB entry.
+        '''
+        friend = ''
+        try:
+            # Get the friend command
+            action = sys.argv[2]
+        except IndexError:
+            logger.info('Syntax: friend add/remove/list [address]')
+        else:
+            action = action.lower()
+            if action == 'list':
+                # List out peers marked as our friend
+                for friend in self.onionrCore.listPeers(randomOrder=False, trust=1):
+                    if friend == self.onionrCore._crypto.pubKey: # do not list our key
+                        continue
+                    friendProfile = onionrusers.OnionrUser(self.onionrCore, friend)
+                    logger.info(friend + ' - ' + friendProfile.getName())
+            elif action in ('add', 'remove'):
+                try:
+                    friend = sys.argv[3]
+                    if not self.onionrUtils.validatePubKey(friend):
+                        raise onionrexceptions.InvalidPubkey('Public key is invalid')
+                    if friend not in self.onionrCore.listPeers():
+                        raise onionrexceptions.KeyNotKnown
+                    friend = onionrusers.OnionrUser(self.onionrCore, friend)
+                except IndexError:
+                    logger.error('Friend ID is required.')
+                except onionrexceptions.KeyNotKnown:
+                    logger.error('That peer is not in our database')
+                else:
+                    if action == 'add':
+                        friend.setTrust(1)
+                        logger.info('Added %s as friend.' % (friend.publicKey,))
+                    else:
+                        friend.setTrust(0)
+                        logger.info('Removed %s as friend.' % (friend.publicKey,))
+            else:
+                logger.info('Syntax: friend add/remove/list [address]')
+
+
+    def banBlock(self):
+        try:
+            ban = sys.argv[2]
+        except IndexError:
+            ban = logger.readline('Enter a block hash:')
+        if self.onionrUtils.validateHash(ban):
+            if not self.onionrCore._blacklist.inBlacklist(ban):
+                try:
+                    self.onionrCore._blacklist.addToDB(ban)
+                    self.onionrCore.removeBlock(ban)
+                except Exception as error:
+                    logger.error('Could not blacklist block', error=error)
+                else:
+                    logger.info('Block blacklisted')
+            else:
+                logger.warn('That block is already blacklisted')
+        else:
+            logger.error('Invalid block hash')
+        return
 
     def listConn(self):
         self.onionrCore.daemonQueueAdd('connectedPeers')
@@ -543,22 +611,39 @@ class Onionr:
             Starts the Onionr communication daemon
         '''
         communicatorDaemon = './communicator2.py'
-        if not os.environ.get("WERKZEUG_RUN_MAIN") == "true":
-            if self._developmentMode:
-                logger.warn('DEVELOPMENT MODE ENABLED (THIS IS LESS SECURE!)', timestamp = False)
-            net = NetController(config.get('client.port', 59496))
-            logger.info('Tor is starting...')
-            if not net.startTor():
-                sys.exit(1)
-            logger.info('Started .onion service: ' + logger.colors.underline + net.myID)
-            logger.info('Our Public key: ' + self.onionrCore._crypto.pubKey)
-            time.sleep(1)
-            #TODO make runable on windows
-            subprocess.Popen([communicatorDaemon, "run", str(net.socksPort)])
-            logger.debug('Started communicator')
-            events.event('daemon_start', onionr = self)
-        self.api = api.API(self.debug)
 
+        apiThread = Thread(target=api.API, args=(self.debug,))
+        apiThread.start()
+        try:
+            time.sleep(3)
+        except KeyboardInterrupt:
+            logger.info('Got keyboard interrupt')
+            time.sleep(1)
+            self.onionrUtils.localCommand('shutdown')
+        else:
+            if apiThread.isAlive():
+                if self._developmentMode:
+                    logger.warn('DEVELOPMENT MODE ENABLED (THIS IS LESS SECURE!)', timestamp = False)
+                net = NetController(config.get('client.port', 59496))
+                logger.info('Tor is starting...')
+                if not net.startTor():
+                    sys.exit(1)
+                logger.info('Started .onion service: ' + logger.colors.underline + net.myID)
+                logger.info('Our Public key: ' + self.onionrCore._crypto.pubKey)
+                time.sleep(1)
+                #TODO make runable on windows
+                subprocess.Popen([communicatorDaemon, "run", str(net.socksPort)])
+                # Print nice header thing :)
+                if config.get('general.display_header', True):
+                    self.header()
+                logger.debug('Started communicator')
+                events.event('daemon_start', onionr = self)
+                try:
+                    while True:
+                        time.sleep(5)
+                except KeyboardInterrupt:
+                    self.onionrCore.daemonQueueAdd('shutdown')
+                    self.onionrUtils.localCommand('shutdown')
         return
 
     def killDaemon(self):
@@ -722,5 +807,12 @@ class Onionr:
         print('Opening %s ...' % url)
         webbrowser.open(url, new = 1, autoraise = True)
 
+    def header(self, message = logger.colors.fg.pink + logger.colors.bold + 'Onionr' + logger.colors.reset + logger.colors.fg.pink + ' has started.'):
+        if os.path.exists('static-data/header.txt'):
+            with open('static-data/header.txt', 'rb') as file:
+                # only to stdout, not file or log or anything
+                sys.stderr.write(file.read().decode().replace('P', logger.colors.fg.pink).replace('W', logger.colors.reset + logger.colors.bold).replace('G', logger.colors.fg.green).replace('\n', logger.colors.reset + '\n').replace('B', logger.colors.bold).replace('V', ONIONR_VERSION))
+                logger.info(logger.colors.fg.lightgreen + '-> ' + str(message) + logger.colors.reset + logger.colors.fg.lightgreen + ' <-\n')
+
 if __name__ == "__main__":
     Onionr()

onionr/onionrblacklist.py

@@ -0,0 +1,115 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This file handles maintenence of a blacklist database, for blocks and peers
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sqlite3, os, logger
+class OnionrBlackList:
+    def __init__(self, coreInst):
+        self.blacklistDB = 'data/blacklist.db'
+        self._core = coreInst
+        
+        if not os.path.exists(self.blacklistDB):
+            self.generateDB()
+        return
+    
+    def inBlacklist(self, data):
+        hashed = self._core._utils.bytesToStr(self._core._crypto.sha3Hash(data))
+        retData = False
+        if not hashed.isalnum():
+            raise Exception("Hashed data is not alpha numeric")
+
+        for i in self._dbExecute("select * from blacklist where hash='%s'" % (hashed,)):
+            retData = True # this only executes if an entry is present by that hash
+            break
+        return retData
+
+    def _dbExecute(self, toExec):
+        conn = sqlite3.connect(self.blacklistDB)
+        c = conn.cursor()
+        retData = c.execute(toExec)
+        conn.commit()
+        return retData
+    
+    def deleteBeforeDate(self, date):
+        # TODO, delete blacklist entries before date
+        return
+    
+    def deleteExpired(self, dataType=0):
+        '''Delete expired entries'''
+        deleteList = []
+        curTime = self._core._utils.getEpoch()
+
+        try:
+            int(dataType)
+        except AttributeError:
+            raise TypeError("dataType must be int")
+
+        for i in self._dbExecute('select * from blacklist where dataType=%s' % (dataType,)):
+            if i[1] == dataType:
+                if (curTime - i[2]) >= i[3]:
+                    deleteList.append(i[0])
+        
+        for thing in deleteList:
+            self._dbExecute("delete from blacklist where hash='%s'" % (thing,))
+
+    def generateDB(self):
+        self._dbExecute('''CREATE TABLE blacklist(
+            hash text primary key not null,
+            dataType int,
+            blacklistDate int,
+            expire int
+            );
+        ''')
+        return
+    
+    def clearDB(self):
+        self._dbExecute('''delete from blacklist;);''')
+
+    def getList(self):
+        data = self._dbExecute('select * from blacklist')
+        myList = []
+        for i in data:
+            myList.append(i[0])
+        return myList
+
+    def addToDB(self, data, dataType=0, expire=0):
+        '''Add to the blacklist. Intended to be block hash, block data, peers, or transport addresses
+        0=block
+        1=peer
+        2=pubkey
+        '''
+        # we hash the data so we can remove data entirely from our node's disk
+        hashed = self._core._utils.bytesToStr(self._core._crypto.sha3Hash(data))
+
+        if self.inBlacklist(hashed):
+            return
+
+        if not hashed.isalnum():
+            raise Exception("Hashed data is not alpha numeric")
+        try:
+            int(dataType)
+        except ValueError:
+            raise Exception("dataType is not int")
+        try:
+            int(expire)
+        except ValueError:
+            raise Exception("expire is not int")
+        #TODO check for length sanity
+        insert = (hashed,)
+        blacklistDate = self._core._utils.getEpoch()
+        self._dbExecute("insert into blacklist (hash, dataType, blacklistDate, expire) VALUES('%s', %s, %s, %s);" % (hashed, dataType, blacklistDate, expire))

onionr/onionrblockapi.py

@@ -1,5 +1,5 @@
 '''
-    Onionr - P2P Microblogging Platform & Social network.
+    Onionr - P2P Anonymous Storage Network
 
     This class contains the OnionrBlocks class which is a class for working with Onionr blocks
 '''

onionr/onionrcrypto.py

@@ -155,26 +155,6 @@ class OnionrCrypto:
             decrypted = anonBox.decrypt(data, encoder=encoding)
         return decrypted
 
-    def symmetricPeerEncrypt(self, data, peer):
-        '''Salsa20 encrypt data to peer (with mac)
-            this function does not accept a key, it is a wrapper for encryption with a peer
-        '''
-        key = self._core.getPeerInfo(4)
-        if type(key) != bytes:
-            key = self._core.getPeerInfo(2)
-        encrypted = self.symmetricEncrypt(data, key, encodedKey=True)
-        return encrypted
-
-    def symmetricPeerDecrypt(self, data, peer):
-        '''Salsa20 decrypt data from peer (with mac)
-        this function does not accept a key, it is a wrapper for encryption with a peer
-        '''
-        key = self._core.getPeerInfo(4)
-        if type(key) != bytes:
-            key = self._core.getPeerInfo(2)
-        decrypted = self.symmetricDecrypt(data, key, encodedKey=True)
-        return decrypted
-
     def symmetricEncrypt(self, data, key, encodedKey=False, returnEncoded=True):
         '''Encrypt data to a 32-byte key (Salsa20-Poly1305 MAC)'''
         if encodedKey:
@@ -247,6 +227,10 @@ class OnionrCrypto:
         return result
 
     def sha3Hash(self, data):
+        try:
+            data = data.encode()
+        except AttributeError:
+            pass
         hasher = hashlib.sha3_256()
         hasher.update(data)
         return hasher.hexdigest()

onionr/onionrdaemontools.py

@@ -0,0 +1,105 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    Contains the CommunicatorUtils class which contains useful functions for the communicator daemon
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import onionrexceptions, onionrpeers, onionrproofs, base64, logger
+from dependencies import secrets
+class DaemonTools:
+    def __init__(self, daemon):
+            self.daemon = daemon
+            self.announceCache = {}
+
+    def announceNode(self):
+        '''Announce our node to our peers'''
+        retData = False
+
+        # Announce to random online peers
+        for i in self.daemon.onlinePeers:
+            if not i in self.announceCache:
+                peer = i
+                break
+        else:
+            peer = self.daemon.pickOnlinePeer()
+
+        ourID = self.daemon._core.hsAddress.strip()
+
+        url = 'http://' + peer + '/public/announce/'
+        data = {'node': ourID}
+
+        combinedNodes = ourID + peer
+
+        if peer in self.announceCache:
+            data['random'] = self.announceCache[peer]
+        else:
+            proof = onionrproofs.DataPOW(combinedNodes, forceDifficulty=4)
+            data['random'] = base64.b64encode(proof.waitForResult()[1])
+            self.announceCache[peer] = data['random']
+
+        logger.info('Announcing node to ' + url)
+        if self.daemon._core._utils.doPostRequest(url, data) == 'Success':
+            retData = True
+        self.daemon.decrementThreadCount('announceNode')
+        return retData
+
+    def netCheck(self):
+        '''Check if we are connected to the internet or not when we can't connect to any peers'''
+        if len(self.daemon.onlinePeers) != 0:
+            if not self.daemon._core._utils.checkNetwork(torPort=self.daemon.proxyPort):
+                logger.warn('Network check failed, are you connected to the internet?')
+                self.daemon.isOnline = False
+        self.daemon.decrementThreadCount('netCheck')
+    
+    def cleanOldBlocks(self):
+        '''Delete old blocks if our disk allocation is full/near full'''
+        while self.daemon._core._utils.storageCounter.isFull():
+            oldest = self.daemon._core.getBlockList()[0]
+            self.daemon._core._blacklist.addToDB(oldest)
+            self.daemon._core.removeBlock(oldest)
+            logger.info('Deleted block: %s' % (oldest,))        
+        self.daemon.decrementThreadCount('cleanOldBlocks')
+
+    def cooldownPeer(self):
+        '''Randomly add an online peer to cooldown, so we can connect a new one'''
+        onlinePeerAmount = len(self.daemon.onlinePeers)
+        minTime = 300
+        cooldownTime = 600
+        toCool = ''
+        tempConnectTimes = dict(self.daemon.connectTimes)
+
+        # Remove peers from cooldown that have been there long enough
+        tempCooldown = dict(self.daemon.cooldownPeer)
+        for peer in tempCooldown:
+            if (self.daemon._core._utils.getEpoch() - tempCooldown[peer]) >= cooldownTime:
+                del self.daemon.cooldownPeer[peer]
+
+        # Cool down a peer, if we have max connections alive for long enough
+        if onlinePeerAmount >= self.daemon._core.config.get('peers.maxConnect'):
+            finding = True
+            while finding:
+                try:
+                    toCool = min(tempConnectTimes, key=tempConnectTimes.get)
+                    if (self.daemon._core._utils.getEpoch() - tempConnectTimes[toCool]) < minTime:
+                        del tempConnectTimes[toCool]
+                    else:
+                        finding = False
+                except ValueError:
+                    break
+            else:
+                self.daemon.removeOnlinePeer(toCool)
+                self.daemon.cooldownPeer[toCool] = self.daemon._core._utils.getEpoch()
+        self.daemon.decrementThreadCount('cooldownPeer')

onionr/onionrexceptions.py

@@ -1,5 +1,5 @@
 '''
-    Onionr - P2P Microblogging Platform & Social network.
+    Onionr - P2P Anonymous Storage Network
 
     This file contains exceptions for onionr
 '''
@@ -34,10 +34,19 @@ class OnlinePeerNeeded(Exception):
 class InvalidPubkey(Exception):
     pass
 
+class KeyNotKnown(Exception):
+    pass
+
 # block exceptions
 class InvalidMetadata(Exception):
     pass
 
+class BlacklistedBlock(Exception):
+    pass
+
+class DataExists(Exception):
+    pass
+
 class InvalidHexHash(Exception):
     '''When a string is not a valid hex string of appropriate length for a hash value'''
     pass
@@ -52,3 +61,8 @@ class MissingPort(Exception):
 
 class InvalidAddress(Exception):
     pass
+
+# file exceptions
+
+class DiskAllocationReached(Exception):
+    pass

onionr/onionrpeers.py

@@ -1,5 +1,5 @@
 '''
-    Onionr - P2P Microblogging Platform & Social network.
+    Onionr - P2P Anonymous Storage Network
 
     This file contains both the PeerProfiles class for network profiling of Onionr nodes
 '''
@@ -17,7 +17,7 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import core, config, logger
+import core, config, logger, sqlite3
 class PeerProfiles:
     '''
         PeerProfiles
@@ -72,7 +72,7 @@ def getScoreSortedPeerList(coreInst):
     return peerList
 
 def peerCleanup(coreInst):
-    '''Removes peers who have been offline too long'''
+    '''Removes peers who have been offline too long or score too low'''
     if not type(coreInst is core.Core):
         raise TypeError('coreInst must be instance of core.Core')
 
@@ -89,4 +89,17 @@ def peerCleanup(coreInst):
         # Remove peers that go below the negative score
         if PeerProfiles(address, coreInst).score < minScore:
             coreInst.removeAddress(address)
-            logger.warn('Removed address ' + address + '.')
+            try:
+                if (int(coreInst._utils.getEpoch()) - int(coreInst.getPeerInfo(address, 'dateSeen'))) >= 600:
+                    expireTime = 600
+                else:
+                    expireTime = 86400
+                coreInst._blacklist.addToDB(address, dataType=1, expire=expireTime)
+            except sqlite3.IntegrityError: #TODO just make sure its not a unique constraint issue
+                pass
+            except ValueError:
+                pass
+            logger.warn('Removed address ' + address + '.')
+
+    # Unban probably not malicious peers TODO improve
+    coreInst._blacklist.deleteExpired(dataType=1)

onionr/onionrusers.py

@@ -0,0 +1,75 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    Contains abstractions for interacting with users of Onionr
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import onionrblockapi, logger, onionrexceptions
+class OnionrUser:
+    def __init__(self, coreInst, publicKey):
+        self.trust = 0
+        self._core = coreInst
+        self.publicKey = publicKey
+
+        self.trust = self._core.getPeerInfo(self.publicKey, 'trust')
+        return
+    
+    def setTrust(self, newTrust):
+        '''Set the peers trust. 0 = not trusted, 1 = friend, 2 = ultimate'''
+        self._core.setPeerInfo(self.publicKey, 'trust', newTrust)
+
+    def isFriend(self):
+        if self._core.getPeerInfo(self.publicKey, 'trust') == 1:
+            return True
+        return False
+    
+    def getName(self):
+        retData = 'anonymous'
+        name = self._core.getPeerInfo(self.publicKey, 'name')
+        try:
+            if len(name) > 0:
+                retData = name
+        except ValueError:
+            pass
+        return retData
+
+    def encrypt(self, data):
+        encrypted = coreInst._crypto.pubKeyEncrypt(data, self.publicKey, encodedData=True)
+        return encrypted
+    
+    def decrypt(self, data):
+        decrypted = coreInst._crypto.pubKeyDecrypt(data, self.publicKey, encodedData=True)
+        return decrypted
+    
+    def forwardEncrypt(self, data):
+        return
+    
+    def forwardDecrypt(self, encrypted):
+        return
+    
+    def findAndSetID(self):
+        '''Find any info about the user from existing blocks and cache it to their DB entry'''
+        infoBlocks = []
+        for bHash in self._core.getBlocksByType('userInfo'):
+            block = onionrblockapi.Block(bHash, core=self._core)
+            if block.signer == self.publicKey:
+                if block.verifySig():
+                    newName = block.getMetadata('name')
+                    if newName.isalnum():
+                        logger.info('%s is now using the name %s.' % (self.publicKey, self._core._utils.escapeAnsi(newName)))
+                        self._core.setPeerInfo(self.publicKey, 'name', newName)
+            else:
+                raise onionrexceptions.InvalidPubkey

onionr/onionrutils.py

@@ -23,7 +23,7 @@ import nacl.signing, nacl.encoding
 from onionrblockapi import Block
 import onionrexceptions
 from defusedxml import minidom
-import pgpwords
+import pgpwords, onionrusers, storagecounter
 if sys.version_info < (3, 6):
     try:
         import sha3
@@ -40,10 +40,10 @@ class OnionrUtils:
         self._core = coreInstance
 
         self.timingToken = ''
-
         self.avoidDupe = [] # list used to prevent duplicate requests per peer for certain actions
         self.peerProcessing = {} # dict of current peer actions: peer, actionList
-
+        self.storageCounter = storagecounter.StorageCounter(self._core)
+        config.reload()
         return
 
     def getTimeBypassToken(self):
@@ -95,8 +95,10 @@ class OnionrUtils:
                     except IndexError:
                         logger.warn('No pow token')
                         continue
-                    #powHash = self._core._crypto.blake2bHash(base64.b64decode(key[1]) + self._core._crypto.blake2bHash(key[0].encode()))
+                    try:
-                    value = base64.b64decode(key[1])
+                        value = base64.b64decode(key[1])
+                    except binascii.Error:
+                        continue
                     hashedKey = self._core._crypto.blake2bHash(key[0])
                     powHash = self._core._crypto.blake2bHash(value + hashedKey)
                     try:
@@ -106,6 +108,7 @@ class OnionrUtils:
                     if powHash.startswith(b'0000'):
                         if not key[0] in self._core.listPeers(randomOrder=False) and type(key) != None and key[0] != self._core._crypto.pubKey:
                             if self._core.addPeer(key[0], key[1]):
+                                onionrusers.OnionrUser(self._core, key[0]).findAndSetID()
                                 retVal = True
                             else:
                                 logger.warn("Failed to add key")
@@ -126,10 +129,17 @@ class OnionrUtils:
             retVal = False
             if newAdderList != False:
                 for adder in newAdderList.split(','):
-                    if not adder in self._core.listAdders(randomOrder = False) and adder.strip() != self.getMyAddress():
+                    adder = adder.strip()
+                    if not adder in self._core.listAdders(randomOrder = False) and adder != self.getMyAddress() and not self._core._blacklist.inBlacklist(adder):
+                        if not config.get('tor.v3onions') and len(adder) == 62:
+                            continue
                         if self._core.addAddress(adder):
-                            logger.info('Added %s to db.' % adder, timestamp = True)
+                            # Check if we have the maxmium amount of allowed stored peers
-                            retVal = True
+                            if config.get('peers.maxStoredPeers') > len(self._core.listAdders()):
+                                logger.info('Added %s to db.' % adder, timestamp = True)
+                                retVal = True
+                            else:
+                                logger.warn('Reached the maximum amount of peers in the net database as allowed by your config.')
                     else:
                         pass
                         #logger.debug('%s is either our address or already in our DB' % adder)
@@ -261,9 +271,24 @@ class OnionrUtils:
         if myBlock.isEncrypted:
             myBlock.decrypt()
         blockType = myBlock.getMetadata('type') # we would use myBlock.getType() here, but it is bugged with encrypted blocks
+        signer = self.bytesToStr(myBlock.signer)
         try:
             if len(blockType) <= 10:
                 self._core.updateBlockInfo(blockHash, 'dataType', blockType)
+
+                if blockType == 'userInfo':
+                    if myBlock.verifySig():
+                        peerName = myBlock.getMetadata('name')
+                        try:
+                            if len(peerName) > 20:
+                                raise onionrexceptions.InvalidMetdata('Peer name specified is too large')
+                        except TypeError:
+                            pass
+                        except onionrexceptions.InvalidMetadata:
+                            pass
+                        else:
+                            self._core.setPeerInfo(signer, 'name', peerName)
+                            logger.info('%s is now using the name %s.' % (signer, self.escapeAnsi(peerName)))
         except TypeError:
             pass
 
@@ -333,7 +358,7 @@ class OnionrUtils:
 
         return retVal
 
-    def validateMetadata(self, metadata):
+    def validateMetadata(self, metadata, blockData):
         '''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string'''
         # TODO, make this check sane sizes
         retData = False
@@ -363,7 +388,20 @@ class OnionrUtils:
                         break
             else:
                 # if metadata loop gets no errors, it does not break, therefore metadata is valid
-                retData = True
+                # make sure we do not have another block with the same data content (prevent data duplication and replay attacks)
+                nonce = self._core._utils.bytesToStr(self._core._crypto.sha3Hash(blockData))
+                try:
+                    with open(self._core.dataNonceFile, 'r') as nonceFile:
+                        if nonce in nonceFile.read():
+                            retData = False # we've seen that nonce before, so we can't pass metadata
+                            raise onionrexceptions.DataExists
+                except FileNotFoundError:
+                    retData = True
+                except onionrexceptions.DataExists:
+                    # do not set retData to True, because nonce has been seen before
+                    pass
+                else:
+                    retData = True
         else:
             logger.warn('In call to utils.validateMetadata, metadata must be JSON string or a dictionary object')
 
@@ -553,6 +591,7 @@ class OnionrUtils:
         '''
         Do a get request through a local tor or i2p instance
         '''
+        retData = False
         if proxyType == 'tor':
             if port == 0:
                 raise onionrexceptions.MissingPort('Socks port required for Tor HTTP get request')
@@ -596,6 +635,35 @@ class OnionrUtils:
         else:
             self.powSalt = retData
         return retData
+    
+    def strToBytes(self, data):
+        try:
+            data = data.encode()
+        except AttributeError:
+            pass
+        return data
+    def bytesToStr(self, data):
+        try:
+            data = data.decode()
+        except AttributeError:
+            pass
+        return data
+    
+    def checkNetwork(self, torPort=0):
+        '''Check if we are connected to the internet (through Tor)'''
+        retData = False
+        connectURLs = []
+        try:
+            with open('static-data/connect-check.txt', 'r') as connectTest:
+                connectURLs = connectTest.read().split(',')
+            
+            for url in connectURLs:
+                if self.doGetRequest(url, port=torPort) != False:
+                    retData = True
+                    break
+        except FileNotFoundError:
+            pass
+        return retData
 
 def size(path='.'):
     '''
@@ -621,4 +689,4 @@ def humanSize(num, suffix='B'):
         if abs(num) < 1024.0:
             return "%.1f %s%s" % (num, unit, suffix)
         num /= 1024.0
-    return "%.1f %s%s" % (num, 'Yi', suffix)
+    return "%.1f %s%s" % (num, 'Yi', suffix)

onionr/static-data/default-plugins/cliui/info.json

@@ -0,0 +1,5 @@
+{
+    "name" : "cliui",
+    "version" : "1.0",
+    "author" : "onionr"
+}

onionr/static-data/default-plugins/cliui/main.py

@@ -0,0 +1,133 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This is an interactive menu-driven CLI interface for Onionr
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+# Imports some useful libraries
+import logger, config, threading, time, uuid, subprocess
+from onionrblockapi import Block
+
+plugin_name = 'cliui'
+PLUGIN_VERSION = '0.0.1'
+
+class OnionrCLIUI:
+    def __init__(self, apiInst):
+        self.api = apiInst
+        self.myCore = apiInst.get_core()
+        return
+
+    def subCommand(self, command):
+            try:
+                subprocess.run(["./onionr.py", command])
+            except KeyboardInterrupt:
+                pass
+
+    def refresh(self):
+        for i in range(100):
+            print('')
+
+    def start(self):
+        '''Main CLI UI interface menu'''
+        showMenu = True
+        isOnline = "No"
+        firstRun = True
+
+        if self.myCore._utils.localCommand('ping') == 'pong':
+            firstRun = False
+
+        while showMenu:
+            if firstRun:
+                print("please wait while Onionr starts...")
+                daemon = subprocess.Popen(["./onionr.py", "start"], stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
+                time.sleep(30)
+                firstRun = False
+
+            if self.myCore._utils.localCommand('ping') == 'pong':
+                isOnline = "Yes"
+            else:
+                isOnline = "No"
+
+            print('''
+Daemon Running: ''' + isOnline + '''
+            
+1. Flow (Anonymous public chat, use at your own risk)
+2. Mail (Secure email-like service)
+3. File Sharing
+4. User Settings
+5. Start/Stop Daemon
+6. Quit (Does not shutdown daemon)
+            ''')
+            try:
+                choice = input(">").strip().lower()
+            except (KeyboardInterrupt, EOFError):
+                choice = "quit"
+
+            if choice in ("flow", "1"):
+                self.subCommand("flow")
+            elif choice in ("2", "mail"):
+                self.subCommand("mail")
+            elif choice in ("3", "file sharing", "file"):
+                print("Not supported yet")
+            elif choice in ("4", "user settings", "settings"):
+                try:
+                    self.setName()
+                except (KeyboardInterrupt, EOFError) as e:
+                    pass
+            elif choice in ("5", "daemon"):
+                if isOnline == "Yes":
+                    print("Onionr daemon will shutdown...")
+                    #self.myCore._utils.localCommand("shutdown")
+                    self.myCore.daemonQueueAdd('shutdown')
+                    try:
+                        daemon.kill()
+                    except UnboundLocalError:
+                        pass
+                else:
+                    print("Starting Daemon...")
+                    daemon = subprocess.Popen(["./onionr.py", "start"], stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
+            elif choice in ("6", "quit"):
+                showMenu = False
+            elif choice == "":
+                pass
+            else:
+                print("Invalid choice")
+        return
+
+    def setName(self):
+        try:
+            name = input("Enter your name: ")
+            if name != "":
+                self.myCore.insertBlock("userInfo-" + str(uuid.uuid1()), sign=True, header='userInfo', meta={'name': name})
+        except KeyboardInterrupt:
+            pass
+        return
+
+def on_init(api, data = None):
+    '''
+        This event is called after Onionr is initialized, but before the command
+        inputted is executed. Could be called when daemon is starting or when
+        just the client is running.
+    '''
+
+    # Doing this makes it so that the other functions can access the api object
+    # by simply referencing the variable `pluginapi`.
+    pluginapi = api
+    ui = OnionrCLIUI(api)
+    api.commands.register('interactive', ui.start)
+    api.commands.register_help('interactive', 'Open the CLI interface')
+    return

onionr/static-data/default-plugins/pms/main.py

@@ -21,7 +21,9 @@
 # Imports some useful libraries
 import logger, config, threading, time, readline, datetime
 from onionrblockapi import Block
-import onionrexceptions
+import onionrexceptions, onionrusers
+import locale
+locale.setlocale(locale.LC_ALL, '')
 
 plugin_name = 'pms'
 PLUGIN_VERSION = '0.0.1'
@@ -79,8 +81,19 @@ class OnionrMail:
                     continue
                 blockCount += 1
                 pmBlockMap[blockCount] = blockHash
+                
+                block = pmBlocks[blockHash]
+                senderKey = block.signer
+                try:
+                    senderKey = senderKey.decode()
+                except AttributeError:
+                    pass
+                senderDisplay = onionrusers.OnionrUser(self.myCore, senderKey).getName()
+                if senderDisplay == 'anonymous':
+                    senderDisplay = senderKey
+
                 blockDate = pmBlocks[blockHash].getDate().strftime("%m/%d %H:%M")
-                print('%s. %s: %s' % (blockCount, blockDate, blockHash))
+                print('%s. %s - %s: %s' % (blockCount, blockDate, senderDisplay[:12], blockHash))
 
             try:
                 choice = logger.readline('Enter a block number, -r to refresh, or -q to stop: ').strip().lower()
@@ -106,15 +119,15 @@ class OnionrMail:
                 except KeyError:
                     pass
                 else:
+                    cancel = ''
                     readBlock.verifySig()
-                    print('Message recieved from', readBlock.signer)
+                    print('Message recieved from %s' % (readBlock.signer,))
                     print('Valid signature:', readBlock.validSig)
                     if not readBlock.validSig:
-                        logger.warn('This message has an INVALID signature. Anyone could have sent this message.')
+                        logger.warn('This message has an INVALID signature. ANYONE could have sent this message.')
-                        logger.readline('Press enter to continue to message.')
+                        cancel = logger.readline('Press enter to continue to message, or -q to not open the message (recommended).')
-
+                    if cancel != '-q':
-                    print(draw_border(self.myCore._utils.escapeAnsi(readBlock.bcontent.decode().strip())))
+                        print(draw_border(self.myCore._utils.escapeAnsi(readBlock.bcontent.decode().strip())))
-
         return
     
     def draftMessage(self):

onionr/static-data/default_config.json

@@ -41,7 +41,7 @@
     },
 
     "tor" : {
-
+        "v3onions": false
     },
 
     "i2p":{
@@ -51,14 +51,18 @@
      },
 
     "allocations":{
-        "disk": 9000000000,
+        "disk": 10000000000,
         "netTotal": 1000000000,
-        "blockCache" : 5000000,
+        "blockCache": 5000000,
-        "blockCacheTotal" : 50000000
+        "blockCacheTotal": 50000000
     },
     "peers":{
-        "minimumScore": -4000,
+        "minimumScore": -100,
-        "maxStoredPeers": 100,
+        "maxStoredPeers": 5000,
-        "maxConnect": 3
+        "maxConnect": 10
+    },
+    "timers":{
+        "lookupBlocks": 25,
+        "getBlocks": 30
     }
 }

onionr/static-data/index.html

@@ -1,6 +1,6 @@
 <h1>This is an Onionr Node</h1>
 
-<p>The content on this server is not necessarily created by the server owner, and was not necessarily stored with the owner's knowledge.</p>
+<p>The content on this server is not necessarily created by the server owner, and was not necessarily stored specifically with the owner's knowledge of its contents.</p>
 
 <p>Onionr is a decentralized, distributed data storage system, that anyone can insert data into.</p>
 

onionr/storagecounter.py

@@ -0,0 +1,61 @@
+'''
+    Onionr - P2P Microblogging Platform & Social network.
+
+    Keeps track of how much disk space we're using
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import config
+
+class StorageCounter:
+    def __init__(self, coreInst):
+        self._core = coreInst
+        self.dataFile = self._core.usageFile
+        return
+
+    def isFull(self):
+        retData = False
+        if self._core.config.get('allocations.disk') <= (self.getAmount() + 1000):
+            retData = True
+        return retData
+
+    def _update(self, data):
+        with open(self.dataFile, 'w') as dataFile:
+            dataFile.write(str(data))
+    def getAmount(self):
+        '''Return how much disk space we're using (according to record)'''
+        retData = 0
+        try:
+            with open(self.dataFile, 'r') as dataFile:
+                retData = int(dataFile.read())
+        except FileNotFoundError:
+            pass
+        return retData
+
+    def addBytes(self, amount):
+        '''Record that we are now using more disk space, unless doing so would exceed configured max'''
+        newAmount = amount + self.getAmount()
+        retData = newAmount
+        if newAmount > self._core.config.get('allocations.disk'):
+            retData = False
+        else:
+            self._update(newAmount)
+        return retData
+
+    def removeBytes(self, amount):
+        '''Record that we are now using less disk space'''
+        newAmount = self.getAmount() - amount
+        self._update(newAmount)
+        return newAmount

readme.md

@@ -5,29 +5,39 @@
 
 Anonymous P2P platform, using Tor & I2P.
 
-Major work in progress.
+***Experimental, not safe or easy to use yet***
 
-***THIS SOFTWARE IS NOT USABLE OR SECURE YET.***
+<hr>
 
 **The main repo for this software is at https://gitlab.com/beardog/Onionr/**
 
-**Roadmap/features:**
+
+# Summary
+
+Onionr is a decentralized, peer-to-peer data storage network, designed to be anonymous and resistant to (meta)data analysis and spam.
+
+Onionr can be used for mail, as a social network, instant messenger, file sharing software, or for encrypted group discussion.
+
+# Roadmap/features
 
 Check the [Gitlab Project](https://gitlab.com/beardog/Onionr/milestones/1) to see progress towards the alpha release.
 
+## Core internal features
+
 * [X] Fully p2p/decentralized, no trackers or other single points of failure
-* [X] High level of anonymity
+* [X] End to end encryption of user data
-* [ ] End to end encryption where applicable
 * [X] Optional non-encrypted blocks, useful for blog posts or public file sharing
-* [ ] Easy API system for integration to websites
+* [X] Easy API system for integration to websites
+* [ ] Metadata analysis resistance (being improved)
 
-# Development
 
-This software is in heavy development. If for some reason you want to get involved, get in touch first.
+## Other features
 
-**Onionr API and functionality is subject to non-backwards compatible change during development**
+**Onionr API and functionality is subject to non-backwards compatible change during pre-alpha development**
 
-# Donate
+## Help out
+
+Everyone is welcome to help out. Please get in touch first if you are making non-trivial changes. If you can't help with programming, you can write documentation or guides.
 
 Bitcoin/Bitcoin Cash: 1onion55FXzm6h8KQw3zFw2igpHcV7LPq
 
@@ -35,6 +45,4 @@ Bitcoin/Bitcoin Cash: 1onion55FXzm6h8KQw3zFw2igpHcV7LPq
 
 The Tor Project, I2P developers, and anyone else do not own, create, or endorse this project, and are not otherwise involved.
 
-The badges (besides travis-ci build) are by Maik Ellerbrock is licensed under a Creative Commons Attribution 4.0 International License.
+The badges (besides travis-ci build) are by Maik Ellerbrock is licensed under a Creative Commons Attribution 4.0 International License.
-
-The onion in the Onionr logo is adapted from [this](https://commons.wikimedia.org/wiki/File:Red_Onion_on_White.JPG) image by Colin on Wikimedia under a Creative Commons Attribution-Share Alike 3.0 Unported license. The Onionr logo is under the same license.

requirements.txt

@@ -1,7 +1,7 @@
 urllib3==1.23
 requests==2.18.4
 PyNaCl==1.2.1
-gevent==1.2.2
+gevent==1.3.6
 sha3==0.2.1
 defusedxml==0.5.0
 simple_crypt==4.1.7