Merge branch 'pom' into 'master'

Merge POM

See merge request beardog/Onionr!18

.gitlab-ci.yml

@@ -1,6 +0,0 @@
-test:
-  script:
-  - apt-get update -qy
-  - apt-get install -y python3-dev python3-pip tor
-  - pip3 install -r requirements.txt
-  - make test

.travis.yml

@@ -1,8 +0,0 @@
-language: python
-python:
-        - "3.6.4"
-# install dependencies
-install:
-        - sudo apt install tor
-        - pip install -r requirements.txt
-script: make test

CONTRIBUTING.md

@@ -27,7 +27,7 @@ And most importantly, please be patient. Onionr is an open source project done b
 
 ## Asking Questions
 
-If you need help with Onionr, you can ask in our
+If you need help with Onionr, you can contact the devs (be polite and remember this is a volunteer-driven non-profit project).
 
 ## Contributing Code
 

Makefile

@@ -18,7 +18,7 @@ uninstall:
 	rm -f $(DESTDIR)$(PREFIX)/bin/onionr
 
 test:
-	@./run-linux stop
+	@./onionr.sh stop
 	@sleep 1
 	@rm -rf onionr/data-backup
 	@mv onionr/data onionr/data-backup | true > /dev/null 2>&1
@@ -29,15 +29,15 @@ test:
 soft-reset:
 	@echo "Soft-resetting Onionr..."
 	rm -f onionr/data/blocks/*.dat onionr/data/*.db onionr/data/block-nonces.dat | true > /dev/null 2>&1
-	@./run-linux version | grep -v "Failed" --color=always
+	@./onionr.sh version | grep -v "Failed" --color=always
 
 reset:
 	@echo "Hard-resetting Onionr..."
 	rm -rf onionr/data/ | true > /dev/null 2>&1
 	cd onionr/static-data/www/ui/; rm -rf ./dist; python compile.py
-	#@./RUN-LINUX.sh version | grep -v "Failed" --color=always
+	#@./onionr.sh.sh version | grep -v "Failed" --color=always
 
 plugins-reset:
 	@echo "Resetting plugins..."
 	rm -rf onionr/data/plugins/ | true > /dev/null 2>&1
-	@./run-linux version | grep -v "Failed" --color=always
+	@./onionr.sh version | grep -v "Failed" --color=always

README.md

@@ -1,48 +1,71 @@
-![Onionr logo](./docs/onionr-logo.png)
+<p align="center">
 
-(***experimental, not safe or easy to use yet***)
+<img src="./docs/onionr-logo.png" width='250'>
+
+</p>
+
+<p align="center">
+    Anonymous P2P storage network 🕵️
+</p>
+
+(***pre-alpha & experimental, not well tested or easy to use yet***)
 
 [![Open Source Love](https://badges.frapsoft.com/os/v3/open-source.png?v=103)](https://github.com/ellerbrock/open-source-badges/)
 
-
-Anonymous P2P platform, using Tor & I2P.
-
 <hr>
 
-**The main repo for this software is at https://gitlab.com/beardog/Onionr/**
+**The main repository for this software is at https://gitlab.com/beardog/Onionr/**
-
 
 # Summary
 
-Onionr is a decentralized, peer-to-peer data storage network, designed to be anonymous and resistant to (meta)data analysis and spam.
+Onionr is a decentralized, peer-to-peer data storage network, designed to be anonymous and resistant to (meta)data analysis and spam/disruption.
+
+Onionr stores data in independent packages referred to as 'blocks'. The blocks are synced to all other nodes in the network. Blocks and user IDs cannot be easily proven to have been created by particular nodes (only inferred). Even if there is enough evidence to believe a particular node created a block, nodes still operate behind Tor or I2P and as such are not trivially known to be at a particular IP address.
+
+Users are identified by ed25519 public keys, which can be used to sign blocks or send encrypted data.
 
 Onionr can be used for mail, as a social network, instant messenger, file sharing software, or for encrypted group discussion.
 
-# Roadmap/features
+![Tor stinks slide image](docs/tor-stinks-02.png)
 
-Check the [Gitlab Project](https://gitlab.com/beardog/Onionr/milestones/1) to see progress towards the alpha release.
+## Main Features
-
-## Core internal features
 
 * [X] Fully p2p/decentralized, no trackers or other single points of failure
 * [X] End to end encryption of user data
 * [X] Optional non-encrypted blocks, useful for blog posts or public file sharing
 * [X] Easy API system for integration to websites
-* [ ] Metadata analysis resistance (being improved)
+* [X] Metadata analysis resistance
-
+* [X] Transport agnosticism (no internet required)
-
-## Other features
 
 **Onionr API and functionality is subject to non-backwards compatible change during pre-alpha development**
 
+# Install and Run on Linux
+
+The following applies to Ubuntu Bionic. Other distros may have different package or command names.
+
+* Have python3.5+, python3-pip, Tor (daemon, not browser) installed (python3-dev recommended)
+* Clone the git repo: `$ git clone https://gitlab.com/beardog/onionr`
+* cd into install direction: `$ cd onionr/`
+* Install the Python dependencies ([virtualenv strongly recommended](https://virtualenv.pypa.io/en/stable/userguide/)): `$ pip3 install -r requirements.txt`
+
 ## Help out
 
-Everyone is welcome to help out. Please get in touch first if you are making non-trivial changes. If you can't help with programming, you can write documentation or guides.
+Everyone is welcome to help out. Help is wanted for the following:
 
-Bitcoin/Bitcoin Cash: 1onion55FXzm6h8KQw3zFw2igpHcV7LPq
+* Development (Get in touch first)
+    * Creation of a lib for use from other languages and faster proof-of-work
+    * Android and IOS development
+    * Windows and Mac support
+    * General bug fixes and development of new features
+* Testing
+* Running stable nodes
+* Security review/audit
+
+Bitcoin: [1onion55FXzm6h8KQw3zFw2igpHcV7LPq](bitcoin:1onion55FXzm6h8KQw3zFw2igpHcV7LPq)
+USD: [Ko-Fi](https://www.ko-fi.com/beardogkf)
 
 ## Disclaimer
 
 The Tor Project, I2P developers, and anyone else do not own, create, or endorse this project, and are not otherwise involved.
 
-The badges (besides travis-ci build) are by Maik Ellerbrock is licensed under a Creative Commons Attribution 4.0 International License.
+The 'open source badge' is by Maik Ellerbrock and is licensed under a Creative Commons Attribution 4.0 International License.

docs/whitepaper.md

@@ -5,7 +5,7 @@
 
 # Introduction
 
-The most important thing in the modern world is information. The ability to communicate freely with others. The internet has provided humanity with the ability to spread information globally, but there are many people who try (and sometimes succeed) to stifle the flow of information.
+One of the most important things in the modern world is information. The ability to communicate freely with others is crucial for maintaining personal liberties. The internet has provided humanity with the ability to spread information globally, but there are many people who try (and sometimes succeed) to stifle the flow of information.
 
 Internet censorship comes in many forms, state censorship, corporate consolidation of media, threats of violence, network exploitation (e.g. denial of service attacks).
 
@@ -14,25 +14,22 @@ To prevent censorship or loss of information, these measures must be in place:
 * Resistance to censorship of underlying infrastructure or of network hosts
 
 * Anonymization of users by default
-   * The Inability to violently coerce human users (personal threats/"doxxing", or totalitarian regime censorship)
+   * The Inability to coerce human users (personal threats/"doxxing", or totalitarian regime censorship)
 
-* Economic availability. A system should not rely on a single device to be constantly online, and should not be overly expensive to use. The majority of people in the world own cell phones, but comparatively few own personal computers, particularly in developing countries.
+* Economic availability. A system should not rely on a single device to be constantly online, and should not be overly expensive to use. The majority of people in the world own cell phones, but comparatively few own personal computers, particularly in developing countries. Internet connectivity can be slow or spotty in many areas.
 
 There are many great projects that tackle decentralization and privacy issues, but there are none which tackle all of the above issues. Some of the existing networks have also not worked well in practice, or are more complicated than they need to be.
 
 # Onionr Design Goals
 
-When designing Onionr we had these goals in mind:
+When designing Onionr we had these main goals in mind:
 
 * Anonymous Blocks
-
    * Difficult to determine block creator or users regardless of transport used
-* Default Anonymous Transport Layer
+* Node-anonymity
- * Tor and I2P
 * Transport agnosticism
-* Default global sync, but can configure what blocks to seed
+* Default global sync, but configurable
 * Spam resistance
-* Encrypted blocks
 
 # Onionr Design
 
@@ -40,23 +37,23 @@ When designing Onionr we had these goals in mind:
 
 ## General Overview
 
-At its core, Onionr is merely a description for storing data in self-verifying packages ("blocks"). These blocks can be encrypted to a user (or self), encrypted symmetrically, or not at all. Blocks can be signed by their creator, but regardless, they are self-verifying due to being identified by a sha3-256 hash value; once a block is created, it cannot be modified.
+At its core, Onionr is merely a description for storing data in self-verifying packages ("blocks"). These blocks can be encrypted to a user (or for one's self), encrypted symmetrically, or not at all. Blocks can be signed by their creator, but regardless, they are self-verifying due to being identified by a sha3-256 hash value; once a block is created, it cannot be modified.
 
-Onionr exchanges a list of blocks between all nodes. By default, all nodes download and share all other blocks, however this is configurable.
+Onionr exchanges a list of blocks between all nodes. By default, all nodes download and share all other blocks, however this is configurable. Blocks do not rely on any particular order of receipt or transport mechanism.
 
 ## User IDs
 
 User IDs are simply Ed25519 public keys. They are represented in Base32 format, or encoded using the [PGP Word List](https://en.wikipedia.org/wiki/PGP_word_list).
 
-Public keys can be generated deterministicly with a password using a key derivation function (Argon2id). This password can be shared between many users in order to share data anonymously among a group, using only 1 password. This is useful in some cases, but is risky, as if one user causes the key to be compromised and does not notify the group or revoke the key, there is no way to know.
+Public keys can be generated deterministically with a password using a key derivation function (Argon2id). This password can be shared between many users in order to share data anonymously among a group, using only 1 password. This is useful in some cases, but is risky, as if one user causes the key to be compromised and does not notify the group or revoke the key, there is no way to know.
 
 ## Nodes
 
-Although Onionr is transport agnostic, the only supported transports in the reference implemetation are Tor .onion services and I2P hidden services. Nodes announce their address on creation.
+Although Onionr is transport agnostic, the only supported transports in the reference implementation are Tor .onion services and I2P hidden services. Nodes announce their address on creation by connecting to peers specified in a bootstrap file. Peers in the bootstrap file have no special permissions aside from being default peers.
 
 ### Node Profiling
 
-To mitigate maliciously slow or unreliable nodes, Onionr builds a profile on nodes it connects to. Nodes are assigned a score, which raises based on the amount of successful block transfers, speed, and reliabilty of a node, and reduces based on how unreliable a node is. If a node is unreachable for over 24 hours after contact, it is forgotten. Onionr can also prioritize connection to 'friend' nodes.
+To mitigate maliciously slow or unreliable nodes, Onionr builds a profile on nodes it connects to. Nodes are assigned a score, which raises based on the amount of successful block transfers, speed, and reliability of a node, and reduces the score based on how unreliable a node is. If a node is unreachable for over 24 hours after contact, it is forgotten. Onionr can also prioritize connection to 'friend' nodes.
 
 ## Block Format
 
@@ -90,8 +87,10 @@ Onionr can provide evidence of when a block was inserted by requesting other use
 
 This can be done either by the creator of the block prior to generation, or by any node after insertion.
 
-In addition, randomness beacons such as the one operated by [NIST](https://beacon.nist.gov/home) or the hash of the latest blocks in a cryptocurrency network could be used to affirm that a block was at least not *created* before a given time.
+In addition, randomness beacons such as the one operated by [NIST](https://beacon.nist.gov/home), [Chile](https://beacon.clcert.cl/), or the hash of the latest blocks in a cryptocurrency network could be used to affirm that a block was at least not *created* before a given time.
 
 # Direct Connections
 
-We propose a system to 
+We propose a method of using Onionr's block sync system to enable direct connections between peers by having one peer request to connect to another using the peer's public key. Since the request is within a standard block, proof of work must be used to request connection. If the requested peer is available and wishes to accept the connection,Onionr will generate a temporary .onion address for the other peer to connect to. Alternatively, a reverse connection may be formed, which is faster to establish but requires a message brokering system instead of a standard socket.
+
+The benefits of such a system are increased privacy, and the ability to anonymously communicate from multiple devices at once. In a traditional onion service, one's online status can be monitored and more easily correlated.

onionr/api.py

@@ -17,14 +17,28 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import flask
+from gevent.pywsgi import WSGIServer, WSGIHandler
+from gevent import Timeout
+#import gevent.monkey
+#gevent.monkey.patch_socket()
+import flask, cgi
 from flask import request, Response, abort, send_from_directory
-from gevent.pywsgi import WSGIServer
+import sys, random, threading, hmac, hashlib, base64, time, math, os, json, socket
-import sys, random, threading, hmac, hashlib, base64, time, math, os, json
 import core
 from onionrblockapi import Block
 import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config, onionr
 
+class FDSafeHandler(WSGIHandler):
+    def handle(self):
+       timeout = Timeout(60, exception=Exception)
+       timeout.start()
+
+       #timeout = gevent.Timeout.start_new(3)
+       try:
+           WSGIHandler.handle(self)
+       except Timeout as ex:
+           raise
+
 def guessMime(path):
     '''
         Guesses the mime type of a file from the input filename
@@ -48,8 +62,18 @@ def setBindIP(filePath):
     hostOctets = [str(127), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF))]
     data = '.'.join(hostOctets)
     
+    # Try to bind IP. Some platforms like Mac block non normal 127.x.x.x
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    try:
+        s.bind((data, 0))
+    except OSError:
+        logger.warn('Your platform appears to not support random local host addresses 127.x.x.x. Falling back to 127.0.0.1.')
+        data = '127.0.0.1'
+    s.close()
+    
     with open(filePath, 'w') as bindFile:
         bindFile.write(data)
+
     return data
 
 class PublicAPI:
@@ -70,6 +94,9 @@ class PublicAPI:
         @app.before_request
         def validateRequest():
             '''Validate request has the correct hostname'''
+            # If high security level, deny requests to public
+            if config.get('general.security_level', default=0) > 0:
+                abort(403)
             if type(self.torAdder) is None and type(self.i2pAdder) is None:
                 # abort if our hs addresses are not known
                 abort(403)
@@ -84,6 +111,7 @@ class PublicAPI:
             resp.headers['X-Frame-Options'] = 'deny'
             resp.headers['X-Content-Type-Options'] = "nosniff"
             resp.headers['X-API'] = onionr.API_VERSION
+            resp.headers['Connection'] = "close"
             return resp
 
         @app.route('/')
@@ -97,7 +125,8 @@ class PublicAPI:
 
         @app.route('/getblocklist')
         def getBlockList():
-            bList = clientAPI._core.getBlockList()
+            dateAdjust = request.args.get('date')
+            bList = clientAPI._core.getBlockList(dateRec=dateAdjust)
             for b in self.hideBlocks:
                 if b in bList:
                     bList.remove(b)
@@ -109,9 +138,9 @@ class PublicAPI:
             data = name
             if clientAPI._utils.validateHash(data):
                 if data not in self.hideBlocks:
-                    if os.path.exists(clientAPI._core.dataDir + 'blocks/' + data + '.dat'):
+                    if data in clientAPI._core.getBlockList():
-                        block = Block(hash=data.encode(), core=clientAPI._core)
+                        block = clientAPI.getBlockData(data, raw=True).encode()
-                        resp = base64.b64encode(block.getRaw().encode()).decode()
+                        resp = base64.b64encode(block).decode()
             if len(resp) == 0:
                 abort(404)
                 resp = ""
@@ -133,9 +162,9 @@ class PublicAPI:
 
         @app.route('/pex')
         def peerExchange():
-            response = ','.join(clientAPI._core.listAdders())
+            response = ','.join(clientAPI._core.listAdders(recent=3600))
             if len(response) == 0:
-                response = 'none'
+                response = ''
             return Response(response)
         
         @app.route('/announce', methods=['post'])
@@ -204,7 +233,7 @@ class PublicAPI:
             clientAPI._core.refreshFirstStartVars()
             self.torAdder = clientAPI._core.hsAddress
             time.sleep(1)
-        self.httpServer = WSGIServer((self.host, self.bindPort), app, log=None)
+        self.httpServer = WSGIServer((self.host, self.bindPort), app, log=None, handler_class=FDSafeHandler)
         self.httpServer.serve_forever()
 
 class API:
@@ -221,19 +250,23 @@ class API:
             This initilization defines all of the API entry points and handlers for the endpoints and errors
             This also saves the used host (random localhost IP address) to the data folder in host.txt
         '''
-
+        # assert isinstance(onionrInst, onionr.Onionr)
         # configure logger and stuff
         onionr.Onionr.setupConfig('data/', self = self)
 
         self.debug = debug
         self._privateDelayTime = 3
-        self._core = core.Core()
+        self._core = onionrInst.onionrCore
+        self.startTime = self._core._utils.getEpoch()
         self._crypto = onionrcrypto.OnionrCrypto(self._core)
         self._utils = onionrutils.OnionrUtils(self._core)
         app = flask.Flask(__name__)
         bindPort = int(config.get('client.client.port', 59496))
         self.bindPort = bindPort
 
+        # Be extremely mindful of this
+        self.whitelistEndpoints = ('site', 'www', 'onionrhome', 'board', 'boardContent', 'sharedContent', 'mail', 'mailindex')
+
         self.clientToken = config.get('client.webpassword')
         self.timeBypassToken = base64.b16encode(os.urandom(32)).decode()
 
@@ -242,6 +275,8 @@ class API:
         self.host = setBindIP(self._core.privateApiHostFile)
         logger.info('Running api on %s:%s' % (self.host, self.bindPort))
         self.httpServer = ''
+
+        self.queueResponse = {}
         onionrInst.setClientAPIInst(self)
 
         @app.before_request
@@ -249,6 +284,8 @@ class API:
             '''Validate request has set password and is the correct hostname'''
             if request.host != '%s:%s' % (self.host, self.bindPort):
                 abort(403)
+            if request.endpoint in self.whitelistEndpoints:
+                return
             try:
                 if not hmac.compare_digest(request.headers['token'], self.clientToken):
                     abort(403)
@@ -257,28 +294,105 @@ class API:
 
         @app.after_request
         def afterReq(resp):
-            resp.headers["Content-Security-Policy"] =  "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'"
+            #resp.headers["Content-Security-Policy"] =  "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'"
+            resp.headers['Content-Security-Policy'] = "default-src 'none'; script-src 'self'; object-src 'none'; style-src 'self'; img-src 'self'; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'self'"
             resp.headers['X-Frame-Options'] = 'deny'
             resp.headers['X-Content-Type-Options'] = "nosniff"
             resp.headers['X-API'] = onionr.API_VERSION
             resp.headers['Server'] = ''
             resp.headers['Date'] = 'Thu, 1 Jan 1970 00:00:00 GMT' # Clock info is probably useful to attackers. Set to unix epoch.
+            resp.headers['Connection'] = "close"
             return resp
 
+        @app.route('/board/', endpoint='board')
+        def loadBoard():
+            return send_from_directory('static-data/www/board/', "index.html")
+
+        @app.route('/mail/<path:path>', endpoint='mail')
+        def loadMail(path):
+            return send_from_directory('static-data/www/mail/', path)
+        @app.route('/mail/', endpoint='mailindex')
+        def loadMailIndex():
+            return send_from_directory('static-data/www/mail/', 'index.html')
+
+        @app.route('/board/<path:path>', endpoint='boardContent')
+        def boardContent(path):
+            return send_from_directory('static-data/www/board/', path)
+        @app.route('/shared/<path:path>', endpoint='sharedContent')
+        def sharedContent(path):
+            return send_from_directory('static-data/www/shared/', path)
+
+        @app.route('/www/<path:path>', endpoint='www')
+        def wwwPublic(path):
+            if not config.get("www.private.run", True):
+                abort(403)
+            return send_from_directory(config.get('www.private.path', 'static-data/www/private/'), path)
+
+        @app.route('/queueResponseAdd/<name>', methods=['post'])
+        def queueResponseAdd(name):
+            self.queueResponse[name] = request.form['data']
+            return Response('success')
+        
+        @app.route('/queueResponse/<name>')
+        def queueResponse(name):
+            resp = 'failure'
+            try:
+                resp = self.queueResponse[name]
+            except KeyError:
+                pass
+            else:
+                del self.queueResponse[name]
+            return Response(resp)
+            
         @app.route('/ping')
         def ping():
             return Response("pong!")
 
-        @app.route('/')
+        @app.route('/', endpoint='onionrhome')
         def hello():
-            return Response("hello client")
+            return send_from_directory('static-data/www/private/', 'index.html')
         
-        @app.route('/site/<name>')
+        @app.route('/getblocksbytype/<name>')
-        def site():
+        def getBlocksByType(name):
-            bHash = block
+            blocks = self._core.getBlocksByType(name)
+            return Response(','.join(blocks))
+        
+        @app.route('/gethtmlsafeblockdata/<name>')
+        def getSafeData(name):
+            resp = ''
+            if self._core._utils.validateHash(name):
+                try:
+                    resp =  cgi.escape(Block(name).bcontent, quote=True)
+                except TypeError:
+                    pass
+            else:
+                abort(404)
+            return Response(resp)
+        
+        @app.route('/getblockdata/<name>')
+        def getData(name):
+            resp = ""
+            if self._core._utils.validateHash(name):
+                if name in self._core.getBlockList():
+                    try:
+                        resp = self.getBlockData(name, decrypt=True)
+                    except ValueError:
+                        pass
+                else:
+                    abort(404)
+            else:
+                abort(404)
+            return Response(resp)
+
+        @app.route('/site/<name>', endpoint='site')
+        def site(name):
+            bHash = name
             resp = 'Not Found'
             if self._core._utils.validateHash(bHash):
+                try:
                     resp = Block(bHash).bcontent
+                except TypeError:
+                    pass
                 try:
                     resp = base64.b64decode(resp)
                 except:
@@ -306,7 +420,26 @@ class API:
                 pass
             return Response("bye")
 
-        self.httpServer = WSGIServer((self.host, bindPort), app, log=None)
+        @app.route('/shutdownclean')
+        def shutdownClean():
+            # good for calling from other clients
+            self._core.daemonQueueAdd('shutdown')
+            return Response("bye")
+        
+        @app.route('/getstats')
+        def getStats():
+            #return Response("disabled")
+            while True:
+                try:    
+                    return Response(self._core.serializer.getStats())
+                except AttributeError:
+                    pass
+        
+        @app.route('/getuptime')
+        def showUptime():
+            return Response(str(self.getUptime()))
+
+        self.httpServer = WSGIServer((self.host, bindPort), app, log=None, handler_class=FDSafeHandler)
         self.httpServer.serve_forever()
 
     def setPublicAPIInstance(self, inst):
@@ -327,3 +460,29 @@ class API:
                 return True
         except TypeError:
             return False
+
+    def getUptime(self):
+        while True:
+            try:
+                return self._utils.getEpoch - startTime
+            except AttributeError:
+                # Don't error on race condition with startup
+                pass
+    
+    def getBlockData(self, bHash, decrypt=False, raw=False):
+        bl = Block(bHash, core=self._core)
+        if decrypt:
+            bl.decrypt()
+            if bl.isEncrypted and not bl.decrypted:
+                raise ValueError
+
+        if not raw:
+            retData = {'meta':bl.bheader, 'metadata': bl.bmetadata, 'content': bl.bcontent}
+            for x in list(retData.keys()):
+                try:
+                    retData[x] = retData[x].decode()
+                except AttributeError:
+                    pass
+            return json.dumps(retData)
+        else:
+            return bl.raw

onionr/communicator.py

@@ -21,15 +21,17 @@
 '''
 import sys, os, core, config, json, requests, time, logger, threading, base64, onionr, uuid
 import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block
-import onionrdaemontools, onionrsockets, onionrchat, onionr, onionrproofs
+import onionrdaemontools, onionrsockets, onionr, onionrproofs, proofofmemory
 import binascii
 from dependencies import secrets
 from defusedxml import minidom
-
+config.reload()
 class OnionrCommunicatorDaemon:
-    def __init__(self, debug, developmentMode):
+    def __init__(self, onionrInst, proxyPort, developmentMode=config.get('general.dev_mode', False)):
+        onionrInst.communicatorInst = self
         # configure logger and stuff
         onionr.Onionr.setupConfig('data/', self = self)
+        self.proxyPort = proxyPort
 
         self.isOnline = True # Assume we're connected to the internet
 
@@ -37,8 +39,8 @@ class OnionrCommunicatorDaemon:
         self.timers = []
 
         # initalize core with Tor socks port being 3rd argument
-        self.proxyPort = sys.argv[2]
+        self.proxyPort = proxyPort
-        self._core = core.Core(torPort=self.proxyPort)
+        self._core = onionrInst.onionrCore
 
         # intalize NIST beacon salt and time
         self.nistSaltTimestamp = 0
@@ -49,9 +51,6 @@ class OnionrCommunicatorDaemon:
         # loop time.sleep delay in seconds
         self.delay = 1
 
-        # time app started running for info/statistics purposes
-        self.startTime = self._core._utils.getEpoch()
-
         # lists of connected peers and peers we know we can't reach currently
         self.onlinePeers = []
         self.offlinePeers = []
@@ -66,7 +65,7 @@ class OnionrCommunicatorDaemon:
         self.shutdown = False
 
         # list of new blocks to download, added to when new block lists are fetched from peers
-        self.blockQueue = []
+        self.blockQueue = {}
 
         # list of blocks currently downloading, avoid s
         self.currentDownloading = []
@@ -74,6 +73,9 @@ class OnionrCommunicatorDaemon:
         # timestamp when the last online node was seen
         self.lastNodeSeen = None
 
+        # Dict of time stamps for peer's block list lookup times, to avoid downloading full lists all the time
+        self.dbTimestamps = {}
+
         # Clear the daemon queue for any dead messages
         if os.path.exists(self._core.queueDB):
             self._core.clearDaemonQueue()
@@ -81,33 +83,36 @@ class OnionrCommunicatorDaemon:
         # Loads in and starts the enabled plugins
         plugins.reload()
 
+        self.proofofmemory = proofofmemory.ProofOfMemory(self)
+
         # daemon tools are misc daemon functions, e.g. announce to online peers
         # intended only for use by OnionrCommunicatorDaemon
         self.daemonTools = onionrdaemontools.DaemonTools(self)
 
-        self._chat = onionrchat.OnionrChat(self)
+        # time app started running for info/statistics purposes
+        self.startTime = self._core._utils.getEpoch()
 
-        if debug or developmentMode:
+        if developmentMode:
             OnionrCommunicatorTimers(self, self.heartbeat, 30)
 
         # Set timers, function reference, seconds
         # requiresPeer True means the timer function won't fire if we have no connected peers
         peerPoolTimer = OnionrCommunicatorTimers(self, self.getOnlinePeers, 60, maxThreads=1)
-        OnionrCommunicatorTimers(self, self.runCheck, 1)
+        OnionrCommunicatorTimers(self, self.runCheck, 2, maxThreads=1)
         OnionrCommunicatorTimers(self, self.lookupBlocks, self._core.config.get('timers.lookupBlocks'), requiresPeer=True, maxThreads=1)
-        OnionrCommunicatorTimers(self, self.getBlocks, self._core.config.get('timers.getBlocks'), requiresPeer=True)
+        OnionrCommunicatorTimers(self, self.getBlocks, self._core.config.get('timers.getBlocks'), requiresPeer=True, maxThreads=2)
         OnionrCommunicatorTimers(self, self.clearOfflinePeer, 58)
         OnionrCommunicatorTimers(self, self.daemonTools.cleanOldBlocks, 65)
         OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True)
         OnionrCommunicatorTimers(self, self.daemonTools.cooldownPeer, 30, requiresPeer=True)
         OnionrCommunicatorTimers(self, self.uploadBlock, 10, requiresPeer=True, maxThreads=1)
         OnionrCommunicatorTimers(self, self.daemonCommands, 6, maxThreads=1)
-        OnionrCommunicatorTimers(self, self.detectAPICrash, 5, maxThreads=1)
+        OnionrCommunicatorTimers(self, self.detectAPICrash, 30, maxThreads=1)
         deniableBlockTimer = OnionrCommunicatorTimers(self, self.daemonTools.insertDeniableBlock, 180, requiresPeer=True, maxThreads=1)
 
         netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600)
         if config.get('general.security_level') == 0:
-            announceTimer = OnionrCommunicatorTimers(self, self.daemonTools.announceNode, 86400, requiresPeer=True, maxThreads=1)
+            announceTimer = OnionrCommunicatorTimers(self, self.daemonTools.announceNode, 3600, requiresPeer=True, maxThreads=1)
             announceTimer.count = (announceTimer.frequency - 120)
         else:
             logger.debug('Will not announce node.')
@@ -125,8 +130,6 @@ class OnionrCommunicatorDaemon:
             self.socketServer.start()
             self.socketClient = onionrsockets.OnionrSocketClient(self._core)
 
-            # Loads chat messages into memory
-            threading.Thread(target=self._chat.chatHandler).start()
 
         # Main daemon loop, mainly for calling timers, don't do any complex operations here to avoid locking
         try:
@@ -136,6 +139,9 @@ class OnionrCommunicatorDaemon:
                         break
                     i.processTimer()
                 time.sleep(self.delay)
+                # Debug to print out used FDs (regular and net)
+                #proc = psutil.Process()
+                #print(proc.open_files(), len(psutil.net_connections()))
         except KeyboardInterrupt:
             self.shutdown = True
             pass
@@ -164,7 +170,9 @@ class OnionrCommunicatorDaemon:
         existingBlocks = self._core.getBlockList()
         triedPeers = [] # list of peers we've tried this time around
         maxBacklog = 1560 # Max amount of *new* block hashes to have already in queue, to avoid memory exhaustion
+        lastLookupTime = 0 # Last time we looked up a particular peer's list
         for i in range(tryAmount):
+            listLookupCommand = 'getblocklist' # This is defined here to reset it each time
             if len(self.blockQueue) >= maxBacklog:
                 break
             if not self.isOnline:
@@ -186,11 +194,21 @@ class OnionrCommunicatorDaemon:
             triedPeers.append(peer)
             if newDBHash != self._core.getAddressInfo(peer, 'DBHash'):
                 self._core.setAddressInfo(peer, 'DBHash', newDBHash)
+                # Get the last time we looked up a peer's stamp to only fetch blocks since then.
+                # Saved in memory only for privacy reasons
                 try:
-                    newBlocks = self.peerAction(peer, 'getblocklist') # get list of new block hashes
+                    lastLookupTime = self.dbTimestamps[peer]
+                except KeyError:
+                    lastLookupTime = 0
+                else:
+                    listLookupCommand += '?date=%s' % (lastLookupTime,)
+                try:
+                    newBlocks = self.peerAction(peer, listLookupCommand) # get list of new block hashes
                 except Exception as error:
                     logger.warn('Could not get new blocks from %s.' % peer, error = error)
                     newBlocks = False
+                else:
+                    self.dbTimestamps[peer] = self._core._utils.getRoundedEpoch(roundS=60)
                 if newBlocks != False:
                     # if request was a success
                     for i in newBlocks.split('\n'):
@@ -198,15 +216,24 @@ class OnionrCommunicatorDaemon:
                             # if newline seperated string is valid hash
                             if not i in existingBlocks:
                                 # if block does not exist on disk and is not already in block queue
-                                if i not in self.blockQueue and not self._core._blacklist.inBlacklist(i):
+                                if i not in self.blockQueue:
-                                    if onionrproofs.hashMeetsDifficulty(i):
+                                    if onionrproofs.hashMeetsDifficulty(i) and not self._core._blacklist.inBlacklist(i):
-                                        self.blockQueue.append(i) # add blocks to download queue
+                                        if len(self.blockQueue) <= 1000000:
+                                            self.blockQueue[i] = [peer] # add blocks to download queue
+                                else:
+                                    if peer not in self.blockQueue[i]:
+                                        self.blockQueue[i].append(peer)
         self.decrementThreadCount('lookupBlocks')
         return
 
     def getBlocks(self):
         '''download new blocks in queue'''
-        for blockHash in self.blockQueue:
+        for blockHash in list(self.blockQueue):
+            triedQueuePeers = [] # List of peers we've tried for a block
+            try:
+                blockPeers = list(self.blockQueue[blockHash])
+            except KeyError:
+                blockPeers = []
             removeFromQueue = True
             if self.shutdown or not self.isOnline:
                 # Exit loop if shutting down or offline
@@ -217,15 +244,24 @@ class OnionrCommunicatorDaemon:
                 continue
             if blockHash in self._core.getBlockList():
                 logger.debug('Block %s is already saved.' % (blockHash,))
-                self.blockQueue.remove(blockHash)
+                try:
+                    del self.blockQueue[blockHash]
+                except KeyError:
+                    pass
                 continue
             if self._core._blacklist.inBlacklist(blockHash):
                 continue
             if self._core._utils.storageCounter.isFull():
                 break
             self.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block
-            logger.info("Attempting to download %s..." % blockHash)
+            if len(blockPeers) == 0:
                 peerUsed = self.pickOnlinePeer()
+            else:
+                blockPeers = self._core._crypto.randomShuffle(blockPeers)
+                peerUsed = blockPeers.pop(0)
+
+            if not self.shutdown and peerUsed.strip() != '':
+                logger.info("Attempting to download %s from %s..." % (blockHash[:12], peerUsed))
             content = self.peerAction(peerUsed, 'getdata/' + blockHash) # block content from random peer (includes metadata)
             if content != False and len(content) > 0:
                 try:
@@ -247,7 +283,7 @@ class OnionrCommunicatorDaemon:
                     metadata = metas[0]
                     if self._core._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid, and verify nonce
                         if self._core._crypto.verifyPow(content): # check if POW is enough/correct
-                            logger.info('Attempting to save block %s...' % blockHash)
+                            logger.info('Attempting to save block %s...' % blockHash[:12])
                             try:
                                 self._core.setData(content)
                             except onionrexceptions.DiskAllocationReached:
@@ -273,11 +309,15 @@ class OnionrCommunicatorDaemon:
                         pass
                     # Punish peer for sharing invalid block (not always malicious, but is bad regardless)
                     onionrpeers.PeerProfiles(peerUsed, self._core).addScore(-50)
+                    if tempHash != 'ed55e34cb828232d6c14da0479709bfa10a0923dca2b380496e6b2ed4f7a0253':
+                        # Dumb hack for 404 response from peer. Don't log it if 404 since its likely not malicious or a critical error.
                         logger.warn('Block hash validation failed for ' + blockHash + ' got ' + tempHash)
+                    else:
+                        removeFromQueue = False # Don't remove from queue if 404
                 if removeFromQueue:
                     try:
-                        self.blockQueue.remove(blockHash) # remove from block queue both if success or false
+                        del self.blockQueue[blockHash] # remove from block queue both if success or false
-                    except ValueError:
+                    except KeyError:
                         pass
             self.currentDownloading.remove(blockHash)
         self.decrementThreadCount('getBlocks')
@@ -401,6 +441,10 @@ class OnionrCommunicatorDaemon:
             del self.connectTimes[peer]
         except KeyError:
             pass
+        try:
+            del self.dbTimestamps[peer]
+        except KeyError:
+            pass
         try:
             self.onlinePeers.remove(peer)
         except ValueError:
@@ -459,10 +503,12 @@ class OnionrCommunicatorDaemon:
             retData = onionrpeers.PeerProfiles(peer, self._core)
         return retData
 
+    def getUptime(self):
+        return self._core._utils.getEpoch() - self.startTime
+
     def heartbeat(self):
         '''Show a heartbeat debug message'''
-        currentTime = self._core._utils.getEpoch() - self.startTime
+        logger.debug('Heartbeat. Node running for %s.' % self.daemonTools.humanReadableTime(self.getUptime()))
-        logger.debug('Heartbeat. Node running for %s.' % self.daemonTools.humanReadableTime(currentTime))
         self.decrementThreadCount('heartbeat')
 
     def daemonCommands(self):
@@ -470,7 +516,7 @@ class OnionrCommunicatorDaemon:
             Process daemon commands from daemonQueue
         '''
         cmd = self._core.daemonQueue()
-
+        response = ''
         if cmd is not False:
             events.event('daemon_command', onionr = None, data = {'cmd' : cmd})
             if cmd[0] == 'shutdown':
@@ -484,7 +530,11 @@ class OnionrCommunicatorDaemon:
                 logger.debug('Status check; looks good.')
                 open(self._core.dataDir + '.runcheck', 'w+').close()
             elif cmd[0] == 'connectedPeers':
-                self.printOnlinePeers()
+                response = '\n'.join(list(self.onlinePeers)).strip()
+                if response == '':
+                    response = 'none'
+            elif cmd[0] == 'localCommand':
+                response = self._core._utils.localCommand(cmd[1])
             elif cmd[0] == 'pex':
                 for i in self.timers:
                     if i.timerFunction.__name__ == 'lookupAdders':
@@ -505,6 +555,11 @@ class OnionrCommunicatorDaemon:
             else:
                 logger.info('Recieved daemonQueue command:' + cmd[0])
 
+            if cmd[0] not in ('', None):
+                if response != '':
+                    self._core._utils.localCommand('queueResponseAdd/' + cmd[4], post=True, postData={'data': response})
+            response = ''
+
         self.decrementThreadCount('daemonCommands')
 
     def uploadBlock(self):
@@ -512,13 +567,14 @@ class OnionrCommunicatorDaemon:
         # when inserting a block, we try to upload it to a few peers to add some deniability
         triedPeers = []
         finishedUploads = []
+        self.blocksToUpload = self._core._crypto.randomShuffle(self.blocksToUpload)
         if len(self.blocksToUpload) != 0:
             for bl in self.blocksToUpload:
                 if not self._core._utils.validateHash(bl):
                     logger.warn('Requested to upload invalid block')
                     self.decrementThreadCount('uploadBlock')
                     return
-                for i in range(max(len(self.onlinePeers), 2)):
+                for i in range(min(len(self.onlinePeers), 6)):
                     peer = self.pickOnlinePeer()
                     if peer in triedPeers:
                         continue
@@ -534,7 +590,6 @@ class OnionrCommunicatorDaemon:
                     if not self._core._utils.doPostRequest(url, data=data, proxyType=proxyType) == False:
                         self._core._utils.localCommand('waitforshare/' + bl)
                         finishedUploads.append(bl)
-                        break
         for x in finishedUploads:
             try:
                 self.blocksToUpload.remove(x)
@@ -610,18 +665,5 @@ class OnionrCommunicatorTimers:
             self.count = -1 # negative 1 because its incremented at bottom
         self.count += 1
 
-shouldRun = False
+def startCommunicator(onionrInst, proxyPort):
-debug = True
+    OnionrCommunicatorDaemon(onionrInst, proxyPort)
-developmentMode = False
-if config.get('general.dev_mode', True):
-    developmentMode = True
-try:
-    if sys.argv[1] == 'run':
-        shouldRun = True
-except IndexError:
-    pass
-if shouldRun:
-    try:
-        OnionrCommunicatorDaemon(debug, developmentMode)
-    except Exception as e:
-        logger.error('Error occured in Communicator', error = e, timestamp = False)

onionr/config.py

@@ -105,7 +105,8 @@ def check():
             open(get_config_file(), 'a', encoding="utf8").close()
             save()
     except:
-        logger.warn('Failed to check configuration file.')
+        pass
+        #logger.debug('Failed to check configuration file.')
 
 def save():
     '''
@@ -129,7 +130,8 @@ def reload():
         with open(get_config_file(), 'r', encoding="utf8") as configfile:
             set_config(json.loads(configfile.read()))
     except:
-        logger.warn('Failed to parse configuration file.')
+        pass
+        #logger.debug('Failed to parse configuration file.')
 
 def get_config():
     '''

onionr/core.py

@@ -17,12 +17,13 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import sqlite3, os, sys, time, math, base64, tarfile, nacl, logger, json, netcontroller, math, config
+import sqlite3, os, sys, time, math, base64, tarfile, nacl, logger, json, netcontroller, math, config, uuid
 from onionrblockapi import Block
 
-import onionrutils, onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions, onionrvalues
+import onionrutils, onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions
-import onionrblacklist, onionrchat, onionrusers
+import onionrblacklist, onionrusers
-import dbcreator
+import dbcreator, onionrstorage, serializeddata
+from etc import onionrvalues
 
 if sys.version_info < (3, 6):
     try:
@@ -45,10 +46,12 @@ class Core:
             self.dataDir = 'data/'
 
         try:
+            self.onionrInst = None
             self.queueDB = self.dataDir + 'queue.db'
             self.peerDB = self.dataDir + 'peers.db'
             self.blockDB = self.dataDir + 'blocks.db'
             self.blockDataLocation = self.dataDir + 'blocks/'
+            self.blockDataDB = self.blockDataLocation + 'block-data.db'
             self.publicApiHostFile = self.dataDir + 'public-host.txt'
             self.privateApiHostFile = self.dataDir + 'private-host.txt'
             self.addressDB = self.dataDir + 'address.db'
@@ -97,9 +100,11 @@ class Core:
                 logger.warn('Warning: address bootstrap file not found ' + self.bootstrapFileLocation)
 
             self._utils = onionrutils.OnionrUtils(self)
+            self.blockCache = onionrstorage.BlockCache()
             # Initialize the crypto object
             self._crypto = onionrcrypto.OnionrCrypto(self)
             self._blacklist = onionrblacklist.OnionrBlackList(self)
+            self.serializer = serializeddata.SerializedData(self)
 
         except Exception as error:
             logger.error('Failed to initialize core Onionr library.', error=error)
@@ -127,7 +132,7 @@ class Core:
 
         events.event('pubkey_add', data = {'key': peerID}, onionr = None)
 
-        conn = sqlite3.connect(self.peerDB, timeout=10)
+        conn = sqlite3.connect(self.peerDB, timeout=30)
         hashID = self._crypto.pubKeyHashID(peerID)
         c = conn.cursor()
         t = (peerID, name, 'unknown', hashID, 0)
@@ -157,7 +162,7 @@ class Core:
         if type(address) is None or len(address) == 0:
             return False
         if self._utils.validateID(address):
-            conn = sqlite3.connect(self.addressDB, timeout=10)
+            conn = sqlite3.connect(self.addressDB, timeout=30)
             c = conn.cursor()
             # check if address is in database
             # this is safe to do because the address is validated above, but we strip some chars here too just in case
@@ -181,7 +186,7 @@ class Core:
 
             return True
         else:
-            logger.debug('Invalid ID: %s' % address)
+            #logger.debug('Invalid ID: %s' % address)
             return False
 
     def removeAddress(self, address):
@@ -190,7 +195,7 @@ class Core:
         '''
 
         if self._utils.validateID(address):
-            conn = sqlite3.connect(self.addressDB, timeout=10)
+            conn = sqlite3.connect(self.addressDB, timeout=30)
             c = conn.cursor()
             t = (address,)
             c.execute('Delete from adders where address=?;', t)
@@ -210,7 +215,7 @@ class Core:
         '''
 
         if self._utils.validateHash(block):
-            conn = sqlite3.connect(self.blockDB, timeout=10)
+            conn = sqlite3.connect(self.blockDB, timeout=30)
             c = conn.cursor()
             t = (block,)
             c.execute('Delete from hashes where hash=?;', t)
@@ -258,7 +263,7 @@ class Core:
             raise Exception('Block db does not exist')
         if self._utils.hasBlock(newHash):
             return
-        conn = sqlite3.connect(self.blockDB, timeout=10)
+        conn = sqlite3.connect(self.blockDB, timeout=30)
         c = conn.cursor()
         currentTime = self._utils.getEpoch() + self._crypto.secrets.randbelow(301)
         if selfInsert or dataSaved:
@@ -277,6 +282,7 @@ class Core:
             Simply return the data associated to a hash
         '''
 
+        '''
         try:
             # logger.debug('Opening %s' % (str(self.blockDataLocation) + str(hash) + '.dat'))
             dataFile = open(self.blockDataLocation + hash + '.dat', 'rb')
@@ -284,6 +290,8 @@ class Core:
             dataFile.close()
         except FileNotFoundError:
             data = False
+        '''
+        data = onionrstorage.getData(self, hash)
 
         return data
 
@@ -308,10 +316,11 @@ class Core:
             #raise Exception("Data is already set for " + dataHash)
         else:
             if self._utils.storageCounter.addBytes(dataSize) != False:
-                blockFile = open(blockFileName, 'wb')
+                #blockFile = open(blockFileName, 'wb')
-                blockFile.write(data)
+                #blockFile.write(data)
-                blockFile.close()
+                #blockFile.close()
-                conn = sqlite3.connect(self.blockDB, timeout=10)
+                onionrstorage.store(self, data, blockHash=dataHash)
+                conn = sqlite3.connect(self.blockDB, timeout=30)
                 c = conn.cursor()
                 c.execute("UPDATE hashes SET dataSaved=1 WHERE hash = ?;", (dataHash,))
                 conn.commit()
@@ -334,10 +343,10 @@ class Core:
         if not os.path.exists(self.queueDB):
             self.dbCreate.createDaemonDB()
         else:
-            conn = sqlite3.connect(self.queueDB, timeout=10)
+            conn = sqlite3.connect(self.queueDB, timeout=30)
             c = conn.cursor()
             try:
-                for row in c.execute('SELECT command, data, date, min(ID) FROM commands group by id'):
+                for row in c.execute('SELECT command, data, date, min(ID), responseID FROM commands group by id'):
                     retData = row
                     break
             except sqlite3.OperationalError:
@@ -352,34 +361,58 @@ class Core:
 
         return retData
 
-    def daemonQueueAdd(self, command, data=''):
+    def daemonQueueAdd(self, command, data='', responseID=''):
         '''
             Add a command to the daemon queue, used by the communication daemon (communicator.py)
         '''
 
         retData = True
-        # Intended to be used by the web server
 
         date = self._utils.getEpoch()
-        conn = sqlite3.connect(self.queueDB, timeout=10)
+        conn = sqlite3.connect(self.queueDB, timeout=30)
         c = conn.cursor()
-        t = (command, data, date)
+        t = (command, data, date, responseID)
 
         try:
-            c.execute('INSERT INTO commands (command, data, date) VALUES(?, ?, ?)', t)
+            c.execute('INSERT INTO commands (command, data, date, responseID) VALUES(?, ?, ?, ?)', t)
             conn.commit()
-            conn.close()
         except sqlite3.OperationalError:
             retData = False
             self.daemonQueue()
         events.event('queue_push', data = {'command': command, 'data': data}, onionr = None)
+        conn.close()
         return retData
 
+    def daemonQueueGetResponse(self, responseID=''):
+        '''
+            Get a response sent by communicator to the API, by requesting to the API
+        '''
+        assert len(responseID) > 0
+        resp = self._utils.localCommand('queueResponse/' + responseID)
+        return resp
+
+    def daemonQueueWaitForResponse(self, responseID='', checkFreqSecs=1):
+        resp = 'failure'
+        while resp == 'failure':
+            resp = self.daemonQueueGetResponse(responseID)
+            time.sleep(1)
+        return resp
+
+    def daemonQueueSimple(self, command, data='', checkFreqSecs=1):
+        '''
+        A simplified way to use the daemon queue. Will register a command (with optional data) and wait, return the data
+        Not always useful, but saves time + LOC in some cases.
+        This is a blocking function, so be careful.
+        '''
+        responseID = str(uuid.uuid4()) # generate unique response ID
+        self.daemonQueueAdd(command, data=data, responseID=responseID)
+        return self.daemonQueueWaitForResponse(responseID, checkFreqSecs)
+
     def clearDaemonQueue(self):
         '''
             Clear the daemon queue (somewhat dangerous)
         '''
-        conn = sqlite3.connect(self.queueDB, timeout=10)
+        conn = sqlite3.connect(self.queueDB, timeout=30)
         c = conn.cursor()
 
         try:
@@ -393,11 +426,11 @@ class Core:
 
         return
 
-    def listAdders(self, randomOrder=True, i2p=True):
+    def listAdders(self, randomOrder=True, i2p=True, recent=0):
         '''
             Return a list of addresses
         '''
-        conn = sqlite3.connect(self.addressDB, timeout=10)
+        conn = sqlite3.connect(self.addressDB, timeout=30)
         c = conn.cursor()
         if randomOrder:
             addresses = c.execute('SELECT * FROM adders ORDER BY RANDOM();')
@@ -405,8 +438,17 @@ class Core:
             addresses = c.execute('SELECT * FROM adders;')
         addressList = []
         for i in addresses:
+            if len(i[0].strip()) == 0:
+                continue
             addressList.append(i[0])
         conn.close()
+        testList = list(addressList) # create new list to iterate
+        for address in testList:
+            try:
+                if recent > 0 and (self._utils.getEpoch() - self.getAddressInfo(address, 'lastConnect')) > recent:
+                    raise TypeError # If there is no last-connected date or it was too long ago, don't add peer to list if recent is not 0
+            except TypeError:
+                addressList.remove(address)
         return addressList
 
     def listPeers(self, randomOrder=True, getPow=False, trust=0):
@@ -416,7 +458,7 @@ class Core:
             randomOrder determines if the list should be in a random order
             trust sets the minimum trust to list
         '''
-        conn = sqlite3.connect(self.peerDB, timeout=10)
+        conn = sqlite3.connect(self.peerDB, timeout=30)
         c = conn.cursor()
 
         payload = ''
@@ -465,7 +507,7 @@ class Core:
             trust int           4
             hashID text         5
         '''
-        conn = sqlite3.connect(self.peerDB, timeout=10)
+        conn = sqlite3.connect(self.peerDB, timeout=30)
         c = conn.cursor()
 
         command = (peer,)
@@ -491,7 +533,7 @@ class Core:
             Update a peer for a key
         '''
 
-        conn = sqlite3.connect(self.peerDB, timeout=10)
+        conn = sqlite3.connect(self.peerDB, timeout=30)
         c = conn.cursor()
 
         command = (data, peer)
@@ -523,7 +565,7 @@ class Core:
             introduced  10
         '''
 
-        conn = sqlite3.connect(self.addressDB, timeout=10)
+        conn = sqlite3.connect(self.addressDB, timeout=30)
         c = conn.cursor()
 
         command = (address,)
@@ -548,7 +590,7 @@ class Core:
             Update an address for a key
         '''
 
-        conn = sqlite3.connect(self.addressDB, timeout=10)
+        conn = sqlite3.connect(self.addressDB, timeout=30)
         c = conn.cursor()
 
         command = (data, address)
@@ -562,24 +604,27 @@ class Core:
 
         return
 
-    def getBlockList(self, unsaved = False): # TODO: Use unsaved??
+    def getBlockList(self, dateRec = None, unsaved = False):
         '''
             Get list of our blocks
         '''
+        if dateRec == None:
+            dateRec = 0
 
-        conn = sqlite3.connect(self.blockDB, timeout=10)
+        conn = sqlite3.connect(self.blockDB, timeout=30)
         c = conn.cursor()
 
-        if unsaved:
+        # if unsaved:
-            execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();'
+        #     execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();'
-        else:
+        # else:
-            execute = 'SELECT hash FROM hashes ORDER BY dateReceived ASC;'
+        #     execute = 'SELECT hash FROM hashes ORDER BY dateReceived ASC;'
-
+        execute = 'SELECT hash FROM hashes WHERE dateReceived >= ? ORDER BY dateReceived ASC;'
+        args = (dateRec,)
         rows = list()
-        for row in c.execute(execute):
+        for row in c.execute(execute, args):
             for i in row:
                 rows.append(i)
-
+        conn.close()
         return rows
 
     def getBlockDate(self, blockHash):
@@ -587,7 +632,7 @@ class Core:
             Returns the date a block was received
         '''
 
-        conn = sqlite3.connect(self.blockDB, timeout=10)
+        conn = sqlite3.connect(self.blockDB, timeout=30)
         c = conn.cursor()
 
         execute = 'SELECT dateReceived FROM hashes WHERE hash=?;'
@@ -595,7 +640,7 @@ class Core:
         for row in c.execute(execute, args):
             for i in row:
                 return int(i)
-
+        conn.close()
         return None
 
     def getBlocksByType(self, blockType, orderDate=True):
@@ -603,7 +648,7 @@ class Core:
             Returns a list of blocks by the type
         '''
 
-        conn = sqlite3.connect(self.blockDB, timeout=10)
+        conn = sqlite3.connect(self.blockDB, timeout=30)
         c = conn.cursor()
 
         if orderDate:
@@ -617,12 +662,12 @@ class Core:
         for row in c.execute(execute, args):
             for i in row:
                 rows.append(i)
-
+        conn.close()
         return rows
 
     def getExpiredBlocks(self):
         '''Returns a list of expired blocks'''
-        conn = sqlite3.connect(self.blockDB, timeout=10)
+        conn = sqlite3.connect(self.blockDB, timeout=30)
         c = conn.cursor()
         date = int(self._utils.getEpoch())
 
@@ -632,6 +677,7 @@ class Core:
         for row in c.execute(execute):
             for i in row:
                 rows.append(i)
+        conn.close()
         return rows
 
     def setBlockType(self, hash, blockType):
@@ -639,7 +685,7 @@ class Core:
             Sets the type of block
         '''
 
-        conn = sqlite3.connect(self.blockDB, timeout=10)
+        conn = sqlite3.connect(self.blockDB, timeout=30)
         c = conn.cursor()
         c.execute("UPDATE hashes SET dataType = ? WHERE hash = ?;", (blockType, hash))
         conn.commit()
@@ -666,7 +712,7 @@ class Core:
         if key not in ('dateReceived', 'decrypted', 'dataType', 'dataFound', 'dataSaved', 'sig', 'author', 'dateClaimed', 'expire'):
             return False
 
-        conn = sqlite3.connect(self.blockDB, timeout=10)
+        conn = sqlite3.connect(self.blockDB, timeout=30)
         c = conn.cursor()
         args = (data, hash)
         c.execute("UPDATE hashes SET " + key + " = ? where hash = ?;", args)
@@ -675,12 +721,15 @@ class Core:
 
         return True
 
-    def insertBlock(self, data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = {}, expire=None):
+    def insertBlock(self, data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = {}, expire=None, disableForward=False):
         '''
             Inserts a block into the network
             encryptType must be specified to encrypt a block
         '''
-
+        allocationReachedMessage = 'Cannot insert block, disk allocation reached.'
+        if self._utils.storageCounter.isFull():
+            logger.error(allocationReachedMessage)
+            return False
         retData = False
         # check nonce
         dataNonce = self._utils.bytesToStr(self._crypto.sha3Hash(data))
@@ -719,15 +768,17 @@ class Core:
             pass
 
         if encryptType == 'asym':
+            if not disableForward and asymPeer != self._crypto.pubKey:
                 try:
                     forwardEncrypted = onionrusers.OnionrUser(self, asymPeer).forwardEncrypt(data)
                     data = forwardEncrypted[0]
                     meta['forwardEnc'] = True
                 except onionrexceptions.InvalidPubkey:
-                onionrusers.OnionrUser(self, asymPeer).generateForwardKey()
+                    pass
-            onionrusers.OnionrUser(self, asymPeer).generateForwardKey()
+                    #onionrusers.OnionrUser(self, asymPeer).generateForwardKey()
-            fsKey = onionrusers.OnionrUser(self, asymPeer).getGeneratedForwardKeys()[0]
+                fsKey = onionrusers.OnionrUser(self, asymPeer).generateForwardKey()
-            meta['newFSKey'] = fsKey[0]
+                #fsKey = onionrusers.OnionrUser(self, asymPeer).getGeneratedForwardKeys().reverse()
+                meta['newFSKey'] = fsKey
         jsonMeta = json.dumps(meta)
         if sign:
             signature = self._crypto.edSign(jsonMeta.encode() + data, key=self._crypto.privKey, encodeResult=True)
@@ -774,7 +825,12 @@ class Core:
         proof = onionrproofs.POW(metadata, data)
         payload = proof.waitForResult()
         if payload != False:
+            try:
                 retData = self.setData(payload)
+            except onionrexceptions.DiskAllocationReached:
+                logger.error(allocationReachedMessage)
+                retData = False
+            else:
                 # Tell the api server through localCommand to wait for the daemon to upload this block to make stastical analysis more difficult
                 self._utils.localCommand('waitforshare/' + retData)
                 self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
@@ -813,5 +869,4 @@ class Core:
         else:
             logger.error('Onionr daemon is not running.')
             return False
-
         return

onionr/dbcreator.py

@@ -92,7 +92,7 @@ class DBCreator:
             expire int   - block expire date in epoch
         '''
         if os.path.exists(self.core.blockDB):
-            raise Exception("Block database already exists")
+            raise FileExistsError("Block database already exists")
         conn = sqlite3.connect(self.core.blockDB)
         c = conn.cursor()
         c.execute('''CREATE TABLE hashes(
@@ -112,12 +112,25 @@ class DBCreator:
         conn.close()
         return
     
+    def createBlockDataDB(self):
+        if os.path.exists(self.core.blockDataDB):
+            raise FileExistsError("Block data database already exists")
+        conn = sqlite3.connect(self.core.blockDataDB)
+        c = conn.cursor()
+        c.execute('''CREATE TABLE blockData(
+            hash text not null,
+            data blob not null
+            );
+        ''')
+        conn.commit()
+        conn.close()
+
     def createForwardKeyDB(self):
         '''
             Create the forward secrecy key db (*for *OUR* keys*)
         '''
         if os.path.exists(self.core.forwardKeysFile):
-            raise Exception("Block database already exists")
+            raise FileExistsError("Block database already exists")
         conn = sqlite3.connect(self.core.forwardKeysFile)
         c = conn.cursor()
         c.execute('''CREATE TABLE myForwardKeys(
@@ -139,7 +152,6 @@ class DBCreator:
         conn = sqlite3.connect(self.core.queueDB, timeout=10)
         c = conn.cursor()
         # Create table
-        c.execute('''CREATE TABLE commands
+        c.execute('''CREATE TABLE commands (id integer primary key autoincrement, command text, data text, date text, responseID text)''')
-                    (id integer primary key autoincrement, command text, data text, date text)''')
         conn.commit()
         conn.close()

onionr/logger.py

@@ -132,8 +132,11 @@ def raw(data, fd = sys.stdout, sensitive = False):
     if get_settings() & OUTPUT_TO_CONSOLE:
         ts = fd.write('%s\n' % data)
     if get_settings() & OUTPUT_TO_FILE and not sensitive:
+        try:
             with open(_outputfile, "a+") as f:
                 f.write(colors.filter(data) + '\n')
+        except OSError:
+            pass
 
 def log(prefix, data, color = '', timestamp=True, fd = sys.stdout, prompt = True, sensitive = False):
     '''

onionr/netcontroller.py

@@ -22,6 +22,7 @@ import subprocess, os, random, sys, logger, time, signal, config, base64, socket
 from stem.control import Controller
 from onionrblockapi import Block
 from dependencies import secrets
+from shutil import which
 
 def getOpenPort():
     # taken from (but modified) https://stackoverflow.com/a/2838309
@@ -31,6 +32,14 @@ def getOpenPort():
     port = s.getsockname()[1]
     s.close()
     return port
+
+def torBinary():
+    '''Return tor binary path or none if not exists'''
+    torPath = './tor'
+    if not os.path.exists(torPath):
+        torPath = which('tor')
+    return torPath
+
 class NetController:
     '''
         This class handles hidden service setup on Tor and I2P

onionr/onionr.py

@@ -21,18 +21,19 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 import sys
-if sys.version_info[0] == 2 or sys.version_info[1] < 5:
+MIN_PY_VERSION = 6
-    print('Error, Onionr requires Python 3.5+')
+if sys.version_info[0] == 2 or sys.version_info[1] < MIN_PY_VERSION:
+    print('Error, Onionr requires Python 3.%s+' % (MIN_PY_VERSION,))
     sys.exit(1)
 import os, base64, random, getpass, shutil, subprocess, requests, time, platform, datetime, re, json, getpass, sqlite3
-import webbrowser
+import webbrowser, uuid, signal
 from threading import Thread
 import api, core, config, logger, onionrplugins as plugins, onionrevents as events
 import onionrutils
-import netcontroller
+import netcontroller, onionrstorage
 from netcontroller import NetController
 from onionrblockapi import Block
-import onionrproofs, onionrexceptions, onionrusers
+import onionrproofs, onionrexceptions, onionrusers, communicator
 
 try:
     from urllib3.contrib.socks import SOCKSProxyManager
@@ -51,6 +52,7 @@ class Onionr:
             In general, external programs and plugins should not use this class.
         '''
         self.userRunDir = os.getcwd() # Directory user runs the program from
+        self.killed = False
         try:
             os.chdir(sys.path[0])
         except FileNotFoundError:
@@ -66,13 +68,21 @@ class Onionr:
         # Load global configuration data
         data_exists = Onionr.setupConfig(self.dataDir, self = self)
 
+        if netcontroller.torBinary() is None:
+            logger.error('Tor is not installed')
+            sys.exit(1)
+
+        self.communicatorInst = None
         self.onionrCore = core.Core()
+        self.onionrCore.onionrInst = self
         #self.deleteRunFiles()
         self.onionrUtils = onionrutils.OnionrUtils(self.onionrCore)
 
         self.clientAPIInst = '' # Client http api instance
         self.publicAPIInst = '' # Public http api instance
 
+        signal.signal(signal.SIGTERM, self.exitSigterm)
+
         # Handle commands
 
         self.debug = False # Whole application debugging
@@ -177,6 +187,12 @@ class Onionr:
             'add-site': self.addWebpage,
             'addsite': self.addWebpage,
 
+            'openhome': self.openHome,
+            'open-home': self.openHome,
+
+            'export-block': self.exportBlock,
+            'exportblock': self.exportBlock,
+
             'get-file': self.getFile,
             'getfile': self.getFile,
 
@@ -192,7 +208,6 @@ class Onionr:
 
             'ui' : self.openUI,
             'gui' : self.openUI,
-            'chat': self.startChat,
 
             'getpassword': self.printWebPassword,
             'get-password': self.printWebPassword,
@@ -203,8 +218,6 @@ class Onionr:
             'getpasswd': self.printWebPassword,
             'get-passwd': self.printWebPassword,
 
-            'chat': self.startChat,
-
             'friend': self.friendCmd,
             'add-id': self.addID,
             'change-id': self.changeID
@@ -237,7 +250,8 @@ class Onionr:
             'introduce': 'Introduce your node to the public Onionr network',
             'friend': '[add|remove] [public key/id]',
             'add-id': 'Generate a new ID (key pair)',
-            'change-id': 'Change active ID'
+            'change-id': 'Change active ID',
+            'open-home': 'Open your node\'s home/info screen'
         }
 
         # initialize plugins
@@ -253,10 +267,38 @@ class Onionr:
 
         return
     
+    def exitSigterm(self, signum, frame):
+        self.killed = True
+
     '''
         THIS SECTION HANDLES THE COMMANDS
     '''
 
+    def exportBlock(self):
+        exportDir = self.dataDir + 'block-export/'
+        try:
+            assert self.onionrUtils.validateHash(sys.argv[2])
+        except (IndexError, AssertionError):
+            logger.error('No valid block hash specified.')
+            sys.exit(1)
+        else:
+            bHash = sys.argv[2]
+        try:
+            path = sys.argv[3]
+        except (IndexError):
+            if not os.path.exists(exportDir):
+                if os.path.exists(self.dataDir):
+                    os.mkdir(exportDir)
+                else:
+                    logger.error('Onionr not initialized')
+                    sys.exit(1)
+            path = exportDir
+        data = onionrstorage.getData(self.onionrCore, bHash)
+        with open('%s/%s.dat' % (exportDir, bHash), 'wb') as exportFile:
+            exportFile.write(data)
+
+
+
     def showDetails(self):
         details = {
             'Node Address' : self.get_hostname(),
@@ -268,6 +310,14 @@ class Onionr:
         for detail in details:
             logger.info('%s%s: \n%s%s\n' % (logger.colors.fg.lightgreen, detail, logger.colors.fg.green, details[detail]), sensitive = True)
 
+    def openHome(self):
+        try:
+            url = self.onionrUtils.getClientAPIServer()
+        except FileNotFoundError:
+            logger.error('Onionr seems to not be running (could not get api host)')
+        else:
+            webbrowser.open_new_tab('http://%s/#%s' % (url, config.get('client.webpassword')))
+
     def addID(self):
         try:
             sys.argv[2]
@@ -276,7 +326,8 @@ class Onionr:
             newID = self.onionrCore._crypto.keyManager.addKey()[0]
         else:
             logger.warn('Deterministic keys require random and long passphrases.')
-            logger.warn('If a good password is not used, your key can be easily stolen.')
+            logger.warn('If a good passphrase is not used, your key can be easily stolen.')
+            logger.warn('You should use a series of hard to guess words, see this for reference: https://www.xkcd.com/936/')
             pass1 = getpass.getpass(prompt='Enter at least %s characters: ' % (self.onionrCore._crypto.deterministicRequirement,))
             pass2 = getpass.getpass(prompt='Confirm entry: ')
             if self.onionrCore._crypto.safeCompare(pass1, pass2):
@@ -310,14 +361,6 @@ class Onionr:
             else:
                 logger.error('Invalid key %s' % (key,))
 
-    def startChat(self):
-        try:
-            data = json.dumps({'peer': sys.argv[2], 'reason': 'chat'})
-        except IndexError:
-            logger.error('Must specify peer to chat with.')
-        else:
-            self.onionrCore.daemonQueueAdd('startSocket', data)
-
     def getCommands(self):
         return self.cmds
 
@@ -351,46 +394,9 @@ class Onionr:
                 except IndexError:
                     logger.error('Friend ID is required.')
                 except onionrexceptions.KeyNotKnown:
-                    logger.error('That peer is not in our database')
+                    self.onionrCore.addPeer(friend)
-                else:
-                    if action == 'add':
-                        friend.setTrust(1)
-                        logger.info('Added %s as friend.' % (friend.publicKey,))
-                    else:
-                        friend.setTrust(0)
-                        logger.info('Removed %s as friend.' % (friend.publicKey,))
-            else:
-                logger.info('Syntax: friend add/remove/list [address]')
-
-
-    def friendCmd(self):
-        '''List, add, or remove friend(s)
-        Changes their peer DB entry.
-        '''
-        friend = ''
-        try:
-            # Get the friend command
-            action = sys.argv[2]
-        except IndexError:
-            logger.info('Syntax: friend add/remove/list [address]')
-        else:
-            action = action.lower()
-            if action == 'list':
-                # List out peers marked as our friend
-                for friend in self.onionrCore.listPeers(randomOrder=False, trust=1):
-                    if friend == self.onionrCore._crypto.pubKey: # do not list our key
-                        continue
-                    friendProfile = onionrusers.OnionrUser(self.onionrCore, friend)
-                    logger.info(friend + ' - ' + friendProfile.getName())
-            elif action in ('add', 'remove'):
-                try:
-                    friend = sys.argv[3]
-                    if not self.onionrUtils.validatePubKey(friend):
-                        raise onionrexceptions.InvalidPubkey('Public key is invalid')
                     friend = onionrusers.OnionrUser(self.onionrCore, friend)
-                except IndexError:
+                finally:
-                    logger.error('Friend ID is required.')
-                else:
                     if action == 'add':
                         friend.setTrust(1)
                         logger.info('Added %s as friend.' % (friend.publicKey,))
@@ -400,6 +406,15 @@ class Onionr:
             else:
                 logger.info('Syntax: friend add/remove/list [address]')
 
+    def deleteRunFiles(self):
+        try:
+            os.remove(self.onionrCore.publicApiHostFile)
+        except FileNotFoundError:
+            pass
+        try:
+            os.remove(self.onionrCore.privateApiHostFile)
+        except FileNotFoundError:
+            pass
 
     def deleteRunFiles(self):
         try:
@@ -432,7 +447,21 @@ class Onionr:
         return
 
     def listConn(self):
-        self.onionrCore.daemonQueueAdd('connectedPeers')
+        randID = str(uuid.uuid4())
+        self.onionrCore.daemonQueueAdd('connectedPeers', responseID=randID)
+        while True:
+            try:
+                time.sleep(3)
+                peers = self.onionrCore.daemonQueueGetResponse(randID)
+            except KeyboardInterrupt:
+                break
+            if not type(peers) is None:
+                if peers not in ('', 'failure', None):
+                    if peers != False:
+                        print(peers)
+                    else:
+                        print('Daemon probably not running. Unable to list connected peers.')
+                    break
 
     def listPeers(self):
         logger.info('Peer transport address list:')
@@ -720,8 +749,6 @@ class Onionr:
             Starts the Onionr communication daemon
         '''
 
-        communicatorDaemon = './communicator2.py'
-
         # remove runcheck if it exists
         if os.path.isfile('data/.runcheck'):
             logger.debug('Runcheck file found on daemon start, deleting in advance.')
@@ -760,8 +787,12 @@ class Onionr:
         logger.debug('Using public key: %s' % (logger.colors.underline + self.onionrCore._crypto.pubKey))
         time.sleep(1)
 
-        # TODO: make runable on windows
+        self.onionrCore.torPort = net.socksPort
-        communicatorProc = subprocess.Popen([communicatorDaemon, 'run', str(net.socksPort)])
+        communicatorThread = Thread(target=communicator.startCommunicator, args=(self, str(net.socksPort)))
+        communicatorThread.start()
+        
+        while self.communicatorInst is None:
+            time.sleep(0.1)
 
         # print nice header thing :)
         if config.get('general.display_header', True):
@@ -776,17 +807,23 @@ class Onionr:
         events.event('daemon_start', onionr = self)
         try:
             while True:
-                time.sleep(5)
+                time.sleep(3)
-
+                # Debug to print out used FDs (regular and net)
+                #proc = psutil.Process()
+                #print('api-files:',proc.open_files(), len(psutil.net_connections()))
                 # Break if communicator process ends, so we don't have left over processes
-                if communicatorProc.poll() is not None:
+                if self.communicatorInst.shutdown:
                     break
+                if self.killed:
+                    break # Break out if sigterm for clean exit
         except KeyboardInterrupt:
+            pass
+        finally:
             self.onionrCore.daemonQueueAdd('shutdown')
             self.onionrUtils.localCommand('shutdown')
+        net.killTor()
         time.sleep(3)
         self.deleteRunFiles()
-        net.killTor()
         return
 
     def killDaemon(self):
@@ -815,7 +852,7 @@ class Onionr:
 
         try:
             # define stats messages here
-            totalBlocks = len(Block.getBlocks())
+            totalBlocks = len(self.onionrCore.getBlockList())
             signedBlocks = len(Block.getBlocks(signed = True))
             messages = {
                 # info about local client
@@ -938,7 +975,8 @@ class Onionr:
                 logger.error('Block hash is invalid')
                 return
 
-            Block.mergeChain(bHash, fileName)
+            with open(fileName, 'wb') as myFile:
+                myFile.write(base64.b64decode(Block(bHash, core=self.onionrCore).bcontent))
         return
 
     def addWebpage(self):
@@ -961,12 +999,9 @@ class Onionr:
                 return
             logger.info('Adding file... this might take a long time.')
             try:
-                if singleBlock:
                 with open(filename, 'rb') as singleFile:
                     blockhash = self.onionrCore.insertBlock(base64.b64encode(singleFile.read()), header=blockType)
-                else:
+                logger.info('File %s saved in block %s' % (filename, blockhash))
-                    blockhash = Block.createChain(file = filename)
-                logger.info('File %s saved in block %s.' % (filename, blockhash))
             except:
                 logger.error('Failed to save file in block.', timestamp = False)
         else:

onionr/onionrblacklist.py

@@ -82,7 +82,7 @@ class OnionrBlackList:
         return
 
     def clearDB(self):
-        self._dbExecute('''DELETE FROM blacklist;);''')
+        self._dbExecute('''DELETE FROM blacklist;''')
 
     def getList(self):
         data = self._dbExecute('SELECT * FROM blacklist')

onionr/onionrblockapi.py

@@ -1,7 +1,7 @@
 '''
     Onionr - P2P Anonymous Storage Network
 
-    This class contains the OnionrBlocks class which is a class for working with Onionr blocks
+    This file contains the OnionrBlocks class which is a class for working with Onionr blocks
 '''
 '''
     This program is free software: you can redistribute it and/or modify
@@ -19,13 +19,13 @@
 '''
 
 import core as onionrcore, logger, config, onionrexceptions, nacl.exceptions, onionrusers
-import json, os, sys, datetime, base64
+import json, os, sys, datetime, base64, onionrstorage
 
 class Block:
     blockCacheOrder = list() # NEVER write your own code that writes to this!
     blockCache = dict() # should never be accessed directly, look at Block.getCache()
 
-    def __init__(self, hash = None, core = None, type = None, content = None, expire=None):
+    def __init__(self, hash = None, core = None, type = None, content = None, expire=None, decrypt=False):
         # take from arguments
         # sometimes people input a bytes object instead of str in `hash`
         if (not hash is None) and isinstance(hash, bytes):
@@ -51,23 +51,13 @@ class Block:
         self.decrypted = False
         self.signer = None
         self.validSig = False
+        self.autoDecrypt = decrypt
 
         # handle arguments
         if self.getCore() is None:
             self.core = onionrcore.Core()
 
-        # update the blocks' contents if it exists
+        self.update()
-        if not self.getHash() is None:
-            if not self.core._utils.validateHash(self.hash):
-                logger.debug('Block hash %s is invalid.' % self.getHash())
-                raise onionrexceptions.InvalidHexHash('Block hash is invalid.')
-            elif not self.update():
-                logger.debug('Failed to open block %s.' % self.getHash())
-        else:
-            pass
-            #logger.debug('Did not update block.')
-
-    # logic
 
     def decrypt(self, anonymous = True, encodedData = True):
         '''
@@ -91,6 +81,7 @@ class Block:
                 self.bmetadata = json.loads(bmeta)
                 self.signature = core._crypto.pubKeyDecrypt(self.signature, anonymous=anonymous, encodedData=encodedData)
                 self.signer = core._crypto.pubKeyDecrypt(self.signer, anonymous=anonymous, encodedData=encodedData)
+                self.bheader['signer'] = self.signer.decode()
                 self.signedData =  json.dumps(self.bmetadata) + self.bcontent.decode()
                 try:
                     assert self.bmetadata['forwardEnc'] is True
@@ -140,13 +131,15 @@ class Block:
             Outputs:
             - (bool): indicates whether or not the operation was successful
         '''
-
         try:
             # import from string
             blockdata = data
 
             # import from file
             if blockdata is None:
+                blockdata = onionrstorage.getData(self.core, self.getHash()).decode()
+                '''
+                
                 filelocation = file
 
                 readfile = True
@@ -164,13 +157,14 @@ class Block:
                         filelocation = self.core.dataDir + 'blocks/%s.dat' % self.getHash()
 
                 if readfile:
-                    with open(filelocation, 'rb') as f:
+                    blockdata = onionrstorage.getData(self.core, self.getHash()).decode()
-                        blockdata = f.read().decode()
+                    #with open(filelocation, 'rb') as f:
+                    #blockdata = f.read().decode()
 
                 self.blockFile = filelocation
+                '''
             else:
                 self.blockFile = None
-
             # parse block
             self.raw = str(blockdata)
             self.bheader = json.loads(self.getRaw()[:self.getRaw().index('\n')])
@@ -199,6 +193,9 @@ class Block:
             if len(self.getRaw()) <= config.get('allocations.blockCache', 500000):
                 self.cache()
             
+            if self.autoDecrypt:
+                self.decrypt()
+
             return True
         except Exception as e:
             logger.error('Failed to parse block %s.' % self.getHash(), error = e, timestamp = False)
@@ -240,12 +237,15 @@ class Block:
 
         try:
             if self.isValid() is True:
+                '''
                 if (not self.getBlockFile() is None) and (recreate is True):
-                    with open(self.getBlockFile(), 'wb') as blockFile:
+                    onionrstorage.store(self.core, self.getRaw().encode())
-                        blockFile.write(self.getRaw().encode())
+                    #with open(self.getBlockFile(), 'wb') as blockFile:
+                    #    blockFile.write(self.getRaw().encode())
                 else:
-                    self.hash = self.getCore().insertBlock(self.getContent(), header = self.getType(), sign = sign, meta = self.getMetadata(), expire = self.getExpire())
+                '''
-                
+                self.hash = self.getCore().insertBlock(self.getRaw(), header = self.getType(), sign = sign, meta = self.getMetadata(), expire = self.getExpire())
+                if self.hash != False:
                     self.update()
 
                 return self.getHash()
@@ -585,7 +585,7 @@ class Block:
 
         return list()
 
-    def mergeChain(child, file = None, maximumFollows = 32, core = None):
+    def mergeChain(child, file = None, maximumFollows = 1000, core = None):
         '''
             Follows a child Block to its root parent Block, merging content
 
@@ -632,7 +632,7 @@ class Block:
 
             blocks.append(block.getHash())
 
-        buffer = ''
+        buffer = b''
 
         # combine block contents
         for hash in blocks:
@@ -641,100 +641,18 @@ class Block:
             contents = base64.b64decode(contents.encode())
 
             if file is None:
-                buffer += contents.decode()
+                try:
+                    buffer += contents.encode()
+                except AttributeError:
+                    buffer += contents
             else:
                 file.write(contents)
+        if file is not None:
+            file.close()
 
         return (None if not file is None else buffer)
 
-    def createChain(data = None, chunksize = 99800, file = None, type = 'chunk', sign = True, encrypt = False, verbose = False):
+    def exists(bHash):
-        '''
-            Creates a chain of blocks to store larger amounts of data
-
-            The chunksize is set to 99800 because it provides the least amount of PoW for the most amount of data.
-
-            Inputs:
-            - data (*): if `file` is None, the data to be stored in blocks
-            - file (file/str): the filename or file object to read from (or None to read `data` instead)
-            - chunksize (int): the number of bytes per block chunk
-            - type (str): the type header for each of the blocks
-            - sign (bool): whether or not to sign each block
-            - encrypt (str): the public key to encrypt to, or False to disable encryption
-            - verbose (bool): whether or not to return a tuple containing more info
-
-            Outputs:
-            - if `verbose`:
-              - (tuple):
-                - (str): the child block hash
-                - (list): all block hashes associated with storing the file
-            - if not `verbose`:
-              - (str): the child block hash
-        '''
-
-        blocks = list()
-
-        # initial datatype checks
-        if data is None and file is None:
-            return blocks
-        elif not (file is None or (isinstance(file, str) and os.path.exists(file))):
-            return blocks
-        elif isinstance(file, str):
-            file = open(file, 'rb')
-        if not isinstance(data, str):
-            data = str(data)
-
-        if not file is None:
-            filesize = os.stat(file.name).st_size
-            offset = filesize % chunksize
-            maxtimes = int(filesize / chunksize)
-
-            for times in range(0, maxtimes + 1):
-                # read chunksize bytes from the file (end -> beginning)
-                if times < maxtimes:
-                    file.seek(- ((times + 1) * chunksize), 2)
-                    content = file.read(chunksize)
-                else:
-                    file.seek(0, 0)
-                    content = file.read(offset)
-
-                # encode it- python is really bad at handling certain bytes that
-                # are often present in binaries.
-                content = base64.b64encode(content).decode()
-
-                # if it is the end of the file, exit
-                if not content:
-                    break
-
-                # create block
-                block = Block()
-                block.setType(type)
-                block.setContent(content)
-                block.setParent((blocks[-1] if len(blocks) != 0 else None))
-                hash = block.save(sign = sign)
-
-                # remember the hash in cache
-                blocks.append(hash)
-        elif not data is None:
-            for content in reversed([data[n:n + chunksize] for n in range(0, len(data), chunksize)]):
-                # encode chunk with base64
-                content = base64.b64encode(content.encode()).decode()
-
-                # create block
-                block = Block()
-                block.setType(type)
-                block.setContent(content)
-                block.setParent((blocks[-1] if len(blocks) != 0 else None))
-                hash = block.save(sign = sign)
-
-                # remember the hash in cache
-                blocks.append(hash)
-
-        # return different things depending on verbosity
-        if verbose:
-            return (blocks[-1], blocks)
-        return blocks[-1]
-
-    def exists(hash):
         '''
             Checks if a block is saved to file or not
 
@@ -748,15 +666,20 @@ class Block:
         '''
 
         # no input data? scrap it.
-        if hash is None:
+        if bHash is None:
             return False
-
+        '''
         if type(hash) == Block:
             blockfile = hash.getBlockFile()
         else:
             blockfile = onionrcore.Core().dataDir + 'blocks/%s.dat' % hash
+        '''
+        if isinstance(bHash, Block):
+            bHash = bHash.getHash()
         
-        return os.path.exists(blockfile) and os.path.isfile(blockfile)
+        ret = isinstance(onionrstorage.getData(onionrcore.Core(), bHash), type(None))
+
+        return not ret
 
     def getCache(hash = None):
         # give a list of the hashes of the cached blocks
@@ -789,7 +712,7 @@ class Block:
         if block.getHash() in Block.getCache() and not override:
             return False
 
-        # dump old cached blocks if the size exeeds the maximum
+        # dump old cached blocks if the size exceeds the maximum
         if sys.getsizeof(Block.blockCacheOrder) >= config.get('allocations.block_cache_total', 50000000): # 50MB default cache size
             del Block.blockCache[blockCacheOrder.pop(0)]
 

onionr/onionrchat.py

@@ -1,50 +0,0 @@
-'''
-    Onionr - P2P Anonymous Storage Network
-
-    Onionr Chat Messages
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import logger, time
-
-class OnionrChat:
-    def __init__(self, communicatorInst):
-        '''OnionrChat uses onionrsockets (handled by the communicator) to exchange direct chat messages'''
-        self.communicator = communicatorInst
-        self._core = self.communicator._core
-        self._utils = self._core._utils
-
-        self.chats = {} # {'peer': {'date': date, message': message}}
-        self.chatSend = {}
-
-    def chatHandler(self):
-        while not self.communicator.shutdown:
-            for peer in self._core.socketServerConnData:
-                try:
-                    assert self._core.socketReasons[peer] == "chat"
-                except (AssertionError, KeyError) as e:
-                    logger.warn('Peer is not for chat')
-                    continue
-                else:
-                    self.chats[peer] = {'date': self._core.socketServerConnData[peer]['date'], 'data': self._core.socketServerConnData[peer]['data']}
-                    logger.info("CHAT MESSAGE RECIEVED: %s" % self.chats[peer]['data'])
-            for peer in self.communicator.socketClient.sockets:
-                try:
-                    logger.info(self.communicator.socketClient.connPool[peer]['data'])
-                    self.communicator.socketClient.sendData(peer, "lol")
-                except:
-                    pass
-
-            time.sleep(2)

onionr/onionrcrypto.py

@@ -210,12 +210,9 @@ class OnionrCrypto:
         ops = nacl.pwhash.argon2id.OPSLIMIT_SENSITIVE
         mem = nacl.pwhash.argon2id.MEMLIMIT_SENSITIVE
 
-        key = kdf(nacl.secret.SecretBox.KEY_SIZE, passphrase, salt, opslimit=ops, memlimit=mem)
+        key = kdf(32, passphrase, salt, opslimit=ops, memlimit=mem) # Generate seed for ed25519 key
-        key = nacl.public.PrivateKey(key, nacl.encoding.RawEncoder())
+        key = nacl.signing.SigningKey(key)
-        publicKey = key.public_key
+        return (key.verify_key.encode(nacl.encoding.Base32Encoder).decode(), key.encode(nacl.encoding.Base32Encoder).decode())
-
-        return (publicKey.encode(encoder=nacl.encoding.Base32Encoder()),
-        key.encode(encoder=nacl.encoding.Base32Encoder()))
 
     def pubKeyHashID(self, pubkey=''):
         '''Accept a ed25519 public key, return a truncated result of X many sha3_256 hash rounds'''
@@ -269,7 +266,8 @@ class OnionrCrypto:
         except AttributeError:
             pass
         
-        difficulty = math.floor(dataLen / 1000000)
+        difficulty = onionrproofs.getDifficultyForNewBlock(blockContent, ourBlock=False)
+        
         if difficulty < int(config.get('general.minimum_block_pow')):
             difficulty = int(config.get('general.minimum_block_pow'))
         mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode()
@@ -283,5 +281,20 @@ class OnionrCrypto:
 
         return retData
 
-    def safeCompare(self, one, two):
+    @staticmethod
+    def safeCompare(one, two):
         return hmac.compare_digest(one, two)
+        
+    @staticmethod
+    def randomShuffle(theList):
+        myList = list(theList)
+        shuffledList = []
+        myListLength = len(myList) + 1
+        while myListLength > 0:
+            removed = secrets.randbelow(myListLength)
+            try:
+                shuffledList.append(myList.pop(removed))
+            except IndexError:
+                pass
+            myListLength = len(myList)
+        return shuffledList

onionr/onionrdaemontools.py

@@ -34,7 +34,7 @@ class DaemonTools:
         '''Announce our node to our peers'''
         retData = False
         announceFail = False
-        
+        if self.daemon._core.config.get('general.security_level', 0) == 0:
             # Announce to random online peers
             for i in self.daemon.onlinePeers:
                 if not i in self.announceCache:

onionr/onionrexceptions.py

@@ -53,6 +53,9 @@ class BlacklistedBlock(Exception):
 class DataExists(Exception):
     pass
 
+class NoDataAvailable(Exception):
+    pass
+
 class InvalidHexHash(Exception):
     '''When a string is not a valid hex string of appropriate length for a hash value'''
     pass

onionr/onionrfragment.py

@@ -0,0 +1,73 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This file contains the OnionrFragment class which implements the fragment system
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+# onionr:10ch+10ch+10chgdecryptionkey
+import core, sys, binascii, os
+
+FRAGMENT_SIZE = 0.25
+TRUNCATE_LENGTH = 30
+
+class OnionrFragment:
+    def __init__(self, uri=None):
+        uri = uri.replace('onionr:', '')
+        count = 0
+        blocks = []
+        appendData = ''
+        key = ''
+        for x in uri:
+            if x == 'k':
+                key = uri[uri.index('k') + 1:]
+            appendData += x
+            if count == TRUNCATE_LENGTH:
+                blocks.append(appendData)
+                appendData = ''
+                count = 0
+            count += 1
+        self.key = key
+        self.blocks = blocks
+        return
+
+    @staticmethod
+    def generateFragments(data=None, coreInst=None):
+        if coreInst is None:
+            coreInst = core.Core()
+
+        key = os.urandom(32)
+        data = coreInst._crypto.symmetricEncrypt(data, key).decode()
+        blocks = []
+        blockData = b""
+        uri = "onionr:"
+        total = sys.getsizeof(data)
+        for x in data:
+            blockData += x.encode()
+            if round(len(blockData) / len(data), 3) > FRAGMENT_SIZE:
+                blocks.append(core.Core().insertBlock(blockData))
+                blockData = b""
+
+        for bl in blocks:
+            uri += bl[:TRUNCATE_LENGTH]
+        uri += "k"
+        uri += binascii.hexlify(key).decode()
+        return (uri, key)
+
+if __name__ == '__main__':
+    uri = OnionrFragment.generateFragments("test")[0]
+    print(uri)
+    OnionrFragment(uri)

onionr/onionrgui.py

@@ -1,58 +0,0 @@
-#!/usr/bin/env python3
-from tkinter import *
-import core
-class OnionrGUI:
-    def __init__(self):
-        self.dataDir = "/programming/onionr/data/"
-        self.root = Tk()
-        self.root.geometry("450x250")
-        self.core = core.Core()
-        menubar = Menu(self.root)
-
-        # create a pulldown menu, and add it to the menu bar
-        filemenu = Menu(menubar, tearoff=0)
-        filemenu.add_command(label="Open", command=None)
-        filemenu.add_command(label="Save", command=None)
-        filemenu.add_separator()
-        filemenu.add_command(label="Exit", command=self.root.quit)
-        menubar.add_cascade(label="File", menu=filemenu)
-
-        settingsmenu = Menu(menubar, tearoff=0)
-        menubar.add_cascade(label="Settings", menu=settingsmenu)
-
-        helpmenu = Menu(menubar, tearoff=0)
-        menubar.add_cascade(label="Help", menu=helpmenu)
-
-        self.root.config(menu=menubar)
-
-        self.menuFrame = Frame(self.root)
-        self.mainButton = Button(self.menuFrame, text="Main View")
-        self.mainButton.grid(row=0, column=0, padx=0, pady=2, sticky=N+W)
-        self.tabButton1 = Button(self.menuFrame, text="Mail")
-        self.tabButton1.grid(row=0, column=1, padx=0, pady=2, sticky=N+W)
-        self.tabButton2 = Button(self.menuFrame, text="Message Flow")
-        self.tabButton2.grid(row=0, column=3, padx=0, pady=2, sticky=N+W)
-
-        self.menuFrame.grid(row=0, column=0, padx=2, pady=0, sticky=N+W)
-
-
-        self.idFrame = Frame(self.root)
-
-        self.ourIDLabel = Label(self.idFrame, text="ID: ")
-        self.ourIDLabel.grid(row=2, column=0, padx=1, pady=1, sticky=N+W)
-        self.ourID = Entry(self.idFrame)
-        self.ourID.insert(0, self.core._crypto.pubKey)
-        self.ourID.grid(row=2, column=1, padx=1, pady=1, sticky=N+W)
-        self.ourID.config(state='readonly')
-        self.idFrame.grid(row=1, column=0, padx=2, pady=2, sticky=N+W)
-
-        self.syncStatus = Label(self.root, text="Sync Status: 15/100")
-        self.syncStatus.place(relx=1.0, rely=1.0, anchor=S+E)
-        self.peerCount = Label(self.root, text="Connected Peers: 3")
-        self.peerCount.place(relx=0.0, rely=1.0, anchor='sw')
-
-        self.root.wm_title("Onionr")
-        self.root.mainloop()
-        return
-
-OnionrGUI()

onionr/onionrpeers.py

@@ -28,6 +28,7 @@ class PeerProfiles:
         self.friendSigCount = 0
         self.success = 0
         self.failure = 0
+        self.connectTime = None
 
         if not isinstance(coreInst, core.Core):
             raise TypeError("coreInst must be a type of core.Core")
@@ -35,6 +36,7 @@ class PeerProfiles:
         assert isinstance(self.coreInst, core.Core)
 
         self.loadScore()
+        self.getConnectTime()
         return
 
     def loadScore(self):
@@ -45,6 +47,12 @@ class PeerProfiles:
             self.success = 0
         self.score = self.success
     
+    def getConnectTime(self):
+        try:
+            self.connectTime = int(self.coreInst.getAddressInfo(self.address, 'lastConnect'))
+        except (KeyError, ValueError, TypeError) as e:
+            pass
+        
     def saveScore(self):
         '''Save the node's score to the database'''
         self.coreInst.setAddressInfo(self.address, 'success', self.score)
@@ -61,14 +69,20 @@ def getScoreSortedPeerList(coreInst):
 
     peerList = coreInst.listAdders()
     peerScores = {}
+    peerTimes = {}
 
     for address in peerList:
         # Load peer's profiles into a list
         profile = PeerProfiles(address, coreInst)
         peerScores[address] = profile.score
+        if not isinstance(profile.connectTime, type(None)):
+            peerTimes[address] = profile.connectTime
+        else:
+            peerTimes[address] = 9000
 
-    # Sort peers by their score, greatest to least
+    # Sort peers by their score, greatest to least, and then last connected time
     peerList = sorted(peerScores, key=peerScores.get, reverse=True)
+    peerList = sorted(peerTimes, key=peerTimes.get, reverse=True)
     return peerList
 
 def peerCleanup(coreInst):

onionr/onionrproofs.py

@@ -19,7 +19,57 @@
 '''
 
 import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, logger, sys, base64, json
-import core, config
+import core, onionrutils, config
+import onionrblockapi
+
+def getDifficultyModifier(coreOrUtilsInst=None):
+    '''Accepts a core or utils instance returns 
+    the difficulty modifier for block storage based 
+    on a variety of factors, currently only disk use.
+    '''
+    classInst = coreOrUtilsInst
+    retData = 0
+    if isinstance(classInst, core.Core):
+        useFunc = classInst._utils.storageCounter.getPercent
+    elif isinstance(classInst, onionrutils.OnionrUtils):
+        useFunc = classInst.storageCounter.getPercent
+    else:
+        useFunc = core.Core()._utils.storageCounter.getPercent
+
+    percentUse = useFunc()
+
+    if percentUse >= 0.50:
+        retData += 1
+    elif percentUse >= 0.75:
+            retData += 2
+    elif percentUse >= 0.95:
+            retData += 3
+
+    return retData
+
+def getDifficultyForNewBlock(data, ourBlock=True):
+    '''
+    Get difficulty for block. Accepts size in integer, Block instance, or str/bytes full block contents
+    '''
+    retData = 0
+    dataSize = 0
+    if isinstance(data, onionrblockapi.Block):
+        dataSize = len(data.getRaw().encode('utf-8'))
+    elif isinstance(data, str):
+        dataSize = len(data.encode('utf-8'))
+    elif isinstance(data, bytes):
+        dataSize = len(data)
+    elif isinstance(data, int):
+        dataSize = data
+    else:
+        raise ValueError('not Block, str, or int')
+    if ourBlock:
+        minDifficulty = config.get('general.minimum_send_pow')
+    else:
+        minDifficulty = config.get('general.minimum_block_pow')
+
+    retData = max(minDifficulty, math.floor(dataSize / 1000000)) + getDifficultyModifier()
+    return retData
 
 def getHashDifficulty(h):
     '''
@@ -55,6 +105,7 @@ class DataPOW:
         self.difficulty = 0
         self.data = data
         self.threadCount = threadCount
+        self.rounds = 0
         config.reload()
 
         if forceDifficulty == 0:
@@ -96,6 +147,7 @@ class DataPOW:
         while self.hashing:
             rand = nacl.utils.random()
             token = nacl.hash.blake2b(rand + self.data).decode()
+            self.rounds += 1
             #print(token)
             if self.puzzle == token[0:self.difficulty]:
                 self.hashing = False
@@ -106,6 +158,7 @@ class DataPOW:
             endTime = math.floor(time.time())
             if self.reporting:
                 logger.debug('Found token after %s seconds: %s' % (endTime - startTime, token), timestamp=True)
+                logger.debug('Round count: %s' % (self.rounds,))
             self.result = (token, rand)
 
     def shutdown(self):
@@ -146,17 +199,27 @@ class DataPOW:
         return result
 
 class POW:
-    def __init__(self, metadata, data, threadCount = 5):
+    def __init__(self, metadata, data, threadCount = 5, forceDifficulty=0, coreInst=None):
         self.foundHash = False
         self.difficulty = 0
         self.data = data
         self.metadata = metadata
         self.threadCount = threadCount
 
+        try:
+            assert isinstance(coreInst, core.Core)
+        except AssertionError:
+            myCore = core.Core()
+        else:
+            myCore = coreInst
+
         dataLen = len(data) + len(json.dumps(metadata))
-        self.difficulty = math.floor(dataLen / 1000000)
+
-        if self.difficulty <= 2:
+        if forceDifficulty > 0:
-            self.difficulty = int(config.get('general.minimum_block_pow'))
+            self.difficulty = forceDifficulty
+        else:
+            # Calculate difficulty. Dumb for now, may use good algorithm in the future.
+            self.difficulty = getDifficultyForNewBlock(dataLen)
             
         try:
             self.data = self.data.encode()
@@ -168,7 +231,6 @@ class POW:
         self.mainHash = '0' * 64
         self.puzzle = self.mainHash[0:min(self.difficulty, len(self.mainHash))]
 
-        myCore = core.Core()
         for i in range(max(1, threadCount)):
             t = threading.Thread(name = 'thread%s' % i, target = self.pow, args = (True,myCore))
             t.start()

onionr/onionrstorage.py

@@ -0,0 +1,90 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This file handles block storage, providing an abstraction for storing blocks between file system and database
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import core, sys, sqlite3, os, dbcreator
+
+DB_ENTRY_SIZE_LIMIT = 10000 # Will be a config option
+
+class BlockCache:
+    def __init__(self):
+        self.blocks = {}
+    def cleanCache(self):
+        while sys.getsizeof(self.blocks) > 100000000:
+            self.blocks.pop(list(self.blocks.keys())[0])
+
+def dbCreate(coreInst):
+    try:
+        dbcreator.DBCreator(coreInst).createBlockDataDB()
+    except FileExistsError:
+        pass
+
+def _dbInsert(coreInst, blockHash, data):
+    assert isinstance(coreInst, core.Core)
+    dbCreate(coreInst)
+    conn = sqlite3.connect(coreInst.blockDataDB, timeout=10)
+    c = conn.cursor()
+    data = (blockHash, data)
+    c.execute('INSERT INTO blockData (hash, data) VALUES(?, ?);', data)
+    conn.commit()
+    conn.close()
+
+def _dbFetch(coreInst, blockHash):
+    assert isinstance(coreInst, core.Core)
+    dbCreate(coreInst)
+    conn = sqlite3.connect(coreInst.blockDataDB, timeout=10)
+    c = conn.cursor()
+    for i in c.execute('SELECT data from blockData where hash = ?', (blockHash,)):
+        return i[0]
+    conn.commit()
+    conn.close()
+    return None
+
+def store(coreInst, data, blockHash=''):
+    assert isinstance(coreInst, core.Core)
+    assert coreInst._utils.validateHash(blockHash)
+    ourHash = coreInst._crypto.sha3Hash(data)
+    if blockHash != '':
+        assert ourHash == blockHash
+    else:
+        blockHash = ourHash
+    
+    if DB_ENTRY_SIZE_LIMIT >= sys.getsizeof(data):
+        _dbInsert(coreInst, blockHash, data)
+    else:
+        with open('%s/%s.dat' % (coreInst.blockDataLocation, blockHash), 'wb') as blockFile:
+            blockFile.write(data)
+    coreInst.blockCache.cleanCache()
+
+def getData(coreInst, bHash):
+    assert isinstance(coreInst, core.Core)
+    assert coreInst._utils.validateHash(bHash)
+
+    bHash = coreInst._utils.bytesToStr(bHash)
+
+    # First check DB for data entry by hash
+    # if no entry, check disk
+    # If no entry in either, raise an exception
+    retData = None
+    fileLocation = '%s/%s.dat' % (coreInst.blockDataLocation, bHash)
+    if os.path.exists(fileLocation):
+        with open(fileLocation, 'rb') as block:
+            retData = block.read()
+    else:
+        retData = _dbFetch(coreInst, bHash)
+    return retData

onionr/onionrusers.py

@@ -83,7 +83,7 @@ class OnionrUser:
         if self._core._utils.validatePubKey(forwardKey):
             retData = self._core._crypto.pubKeyEncrypt(data, forwardKey, encodedData=True, anonymous=True)
         else:
-            raise onionrexceptions.InvalidPubkey("No valid forward key available for this user")
+            raise onionrexceptions.InvalidPubkey("No valid forward secrecy key available for this user")
         #self.generateForwardKey()
         return (retData, forwardKey)
 
@@ -169,7 +169,9 @@ class OnionrUser:
 
     def addForwardKey(self, newKey, expire=604800):
         if not self._core._utils.validatePubKey(newKey):
-            raise onionrexceptions.InvalidPubkey
+            raise onionrexceptions.InvalidPubkey(newKey)
+        if newKey in self._getForwardKeys():
+            return False
         # Add a forward secrecy key for the peer
         conn = sqlite3.connect(self._core.peerDB, timeout=10)
         c = conn.cursor()

onionr/onionrutils.py

@@ -24,7 +24,8 @@ from onionrblockapi import Block
 import onionrexceptions
 from onionr import API_VERSION
 import onionrevents
-import pgpwords, onionrusers, storagecounter
+import onionrusers, storagecounter
+from etc import pgpwords 
 if sys.version_info < (3, 6):
     try:
         import sha3
@@ -151,27 +152,42 @@ class OnionrUtils:
             logger.error('Failed to read my address.', error = error)
             return None
     
-    def localCommand(self, command, data='', silent = True):
+    def getClientAPIServer(self):
-        '''
+        retData = ''
-            Send a command to the local http API server, securely. Intended for local clients, DO NOT USE for remote peers.
-        '''
-
-        config.reload()
-        self.getTimeBypassToken()
-        # TODO: URL encode parameters, just as an extra measure. May not be needed, but should be added regardless.
-        hostname = ''
-        while hostname == '':
         try:
             with open(self._core.privateApiHostFile, 'r') as host:
                 hostname = host.read()
         except FileNotFoundError:
-                print('wat')
+            raise FileNotFoundError
+        else:
+            retData += '%s:%s' % (hostname, config.get('client.client.port'))
+        return retData
+
+    def localCommand(self, command, data='', silent = True, post=False, postData = {}, maxWait=10):
+        '''
+            Send a command to the local http API server, securely. Intended for local clients, DO NOT USE for remote peers.
+        '''
+        config.reload()
+        self.getTimeBypassToken()
+        # TODO: URL encode parameters, just as an extra measure. May not be needed, but should be added regardless.
+        hostname = ''
+        waited = 0
+        while hostname == '':
+            try:
+                hostname = self.getClientAPIServer()
+            except FileNotFoundError:
                 time.sleep(1)
+                waited += 1
+                if waited == maxWait:
+                    return False
         if data != '':
             data = '&data=' + urllib.parse.quote_plus(data)
-        payload = 'http://%s:%s/%s%s' % (hostname, config.get('client.client.port'), command, data)
+        payload = 'http://%s/%s%s' % (hostname, command, data)
         try:
-            retData = requests.get(payload, headers={'token': config.get('client.webpassword')}).text
+            if post:
+                retData = requests.post(payload, data=postData, headers={'token': config.get('client.webpassword'), 'Connection':'close'}, timeout=(maxWait, 30)).text
+            else:
+                retData = requests.get(payload, headers={'token': config.get('client.webpassword'), 'Connection':'close'}, timeout=(maxWait, 30)).text
         except Exception as error:
             if not silent:
                 logger.error('Failed to make local request (command: %s):%s' % (command, error))
@@ -365,6 +381,7 @@ class OnionrUtils:
         '''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string'''
         # TODO, make this check sane sizes
         retData = False
+        maxClockDifference = 60
 
         # convert to dict if it is json string
         if type(metadata) is str:
@@ -393,13 +410,14 @@ class OnionrUtils:
                         break
                 if i == 'time':
                     if not self.isIntegerString(metadata[i]):
-                        logger.warn('Block metadata time stamp is not integer string')
+                        logger.warn('Block metadata time stamp is not integer string or int')
                         break
-                    if (metadata[i] - self.getEpoch()) > 30:
+                    isFuture = (metadata[i] - self.getEpoch())
-                        logger.warn('Block metadata time stamp is set for the future, which is not allowed.')
+                    if isFuture > maxClockDifference:
+                        logger.warn('Block timestamp is skewed to the future over the max %s: %s' (maxClockDifference, isFuture))
                         break
                     if (self.getEpoch() - metadata[i]) > maxAge:
-                        logger.warn('Block is older than allowed: %s' % (maxAge,))
+                        logger.warn('Block is outdated: %s' % (metadata[i],))
                 elif i == 'expire':
                     try:
                         assert int(metadata[i]) > self.getEpoch()
@@ -443,7 +461,7 @@ class OnionrUtils:
         return retVal
 
     def isIntegerString(self, data):
-        '''Check if a string is a valid base10 integer'''
+        '''Check if a string is a valid base10 integer (also returns true if already an int)'''
         try:
             int(data)
         except ValueError:
@@ -607,7 +625,7 @@ class OnionrUtils:
             proxies = {'http': 'http://127.0.0.1:4444'}
         else:
             return
-        headers = {'user-agent': 'PyOnionr'}
+        headers = {'user-agent': 'PyOnionr', 'Connection':'close'}
         try:
             proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
             r = requests.post(url, data=data, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30))
@@ -632,11 +650,11 @@ class OnionrUtils:
             proxies = {'http': 'http://127.0.0.1:4444'}
         else:
             return
-        headers = {'user-agent': 'PyOnionr'}
+        headers = {'user-agent': 'PyOnionr', 'Connection':'close'}
         response_headers = dict()
         try:
             proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
-            r = requests.get(url, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30))
+            r = requests.get(url, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30), )
             # Check server is using same API version as us
             if not ignoreAPI:
                 try:

onionr/proofofmemory.py

@@ -0,0 +1,29 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This file handles proof of memory functionality
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+class ProofOfMemory:
+    def __init__(self, commInst):
+        self.communicator = commInst
+        return
+    
+    def checkRandomPeer(self):
+        return
+    def checkPeer(self, peer):
+        return

onionr/serializeddata.py

@@ -0,0 +1,43 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This module serializes various data pieces for use in other modules, in particular the web api
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+import core, api, uuid, json
+
+class SerializedData:
+    def __init__(self, coreInst):
+        '''
+        Serialized data is in JSON format:
+        {
+            'success': bool,
+            'foo': 'bar',
+            etc
+        }
+        '''
+        assert isinstance(coreInst, core.Core)
+        self._core = coreInst
+    
+    def getStats(self):
+        '''Return statistics about our node'''
+        stats = {}
+        stats['uptime'] = self._core.onionrInst.communicatorInst.getUptime()
+        stats['connectedNodes'] = '\n'.join(self._core.onionrInst.communicatorInst.onlinePeers)
+        stats['blockCount'] = len(self._core.getBlockList())
+        stats['blockQueueCount'] = len(self._core.onionrInst.communicatorInst.blockQueue)
+        return json.dumps(stats)

onionr/static-data/bootstrap-nodes.txt

@@ -0,0 +1 @@
+dd3llxdp5q6ak3zmmicoy3jnodmroouv2xr7whkygiwp3rl7nf23gdad.onion

onionr/static-data/default-plugins/cliui/main.py

@@ -19,7 +19,7 @@
 '''
 
 # Imports some useful libraries
-import logger, config, threading, time, uuid, subprocess
+import logger, config, threading, time, uuid, subprocess, sys
 from onionrblockapi import Block
 
 plugin_name = 'cliui'
@@ -31,10 +31,13 @@ class OnionrCLIUI:
         self.myCore = apiInst.get_core()
         return
 
-    def subCommand(self, command):
+    def subCommand(self, command, args=None):
             try:
                 #subprocess.run(["./onionr.py", command])
                 #subprocess.Popen(['./onionr.py', command], stdin=subprocess.STD, stdout=subprocess.STDOUT, stderr=subprocess.STDOUT)
+                if args != None:
+                    subprocess.call(['./onionr.py', command, args])
+                else:
                     subprocess.call(['./onionr.py', command])
             except KeyboardInterrupt:
                 pass
@@ -48,12 +51,11 @@ class OnionrCLIUI:
         isOnline = 'No'
         firstRun = True
         choice = ''
-
+        if self.myCore._utils.localCommand('ping', maxWait=10) == 'pong!':
-        if self.myCore._utils.localCommand('ping') == 'pong':
             firstRun = False
 
         while showMenu:
-            if self.myCore._utils.localCommand('ping') == 'pong':
+            if self.myCore._utils.localCommand('ping', maxWait=2) == 'pong!':
                 isOnline = "Yes"
             else:
                 isOnline = "No"
@@ -62,8 +64,7 @@ class OnionrCLIUI:
 1. Flow (Anonymous public chat, use at your own risk)
 2. Mail (Secure email-like service)
 3. File Sharing
-4. User Settings
+4. Quit (Does not shutdown daemon)
-5. Quit (Does not shutdown daemon)
             ''')
             try:
                 choice = input(">").strip().lower()
@@ -75,13 +76,9 @@ class OnionrCLIUI:
             elif choice in ("2", "mail"):
                 self.subCommand("mail")
             elif choice in ("3", "file sharing", "file"):
-                print("Not supported yet")
+                filename = input("Enter full path to file: ").strip()
-            elif choice in ("4", "user settings", "settings"):
+                self.subCommand("addfile", filename)
-                try:
+            elif choice in ("4", "quit"):
-                    self.setName()
-                except (KeyboardInterrupt, EOFError) as e:
-                    pass
-            elif choice in ("5", "quit"):
                 showMenu = False
             elif choice == "":
                 pass
@@ -89,14 +86,6 @@ class OnionrCLIUI:
                 logger.error("Invalid choice")
         return
 
-    def setName(self):
-        try:
-            name = input("Enter your name: ")
-            if name != "":
-                self.myCore.insertBlock("userInfo-" + str(uuid.uuid1()), sign=True, header='userInfo', meta={'name': name})
-        except KeyboardInterrupt:
-            pass
-        return
 
 def on_init(api, data = None):
     '''

onionr/static-data/default-plugins/flow/main.py

@@ -54,9 +54,10 @@ class OnionrFlow:
                     self.flowRunning = False
                 expireTime = self.myCore._utils.getEpoch() + 43200
                 if len(message) > 0:
-                    insertBL = Block(content = message, type = 'txt', expire=expireTime, core = self.myCore)
+                    self.myCore.insertBlock(message, header='txt', expire=expireTime, meta={'ch': self.channel})
-                    insertBL.setMetadata('ch', self.channel)
+                    #insertBL = Block(content = message, type = 'txt', expire=expireTime, core = self.myCore)
-                    insertBL.save()
+                    #insertBL.setMetadata('ch', self.channel)
+                    #insertBL.save()
 
         logger.info("Flow is exiting, goodbye")
         return
@@ -66,10 +67,13 @@ class OnionrFlow:
             time.sleep(1)
         try:
             while self.flowRunning:
-                    for block in Block.getBlocks(type = 'txt', core = self.myCore):
+                    for block in self.myCore.getBlocksByType('txt'):
+                        block = Block(block)
                         if block.getMetadata('ch') != self.channel:
+                            #print('not chan', block.getMetadata('ch'))
                             continue
                         if block.getHash() in self.alreadyOutputed:
+                            #print('already')
                             continue
                         if not self.flowRunning:
                             break

onionr/static-data/default-plugins/metadataprocessor/main.py

@@ -28,24 +28,6 @@ plugin_name = 'metadataprocessor'
 
 # event listeners
 
-def _processUserInfo(api, newBlock):
-    '''
-        Set the username for a particular user, from a signed block by them
-    '''
-    myBlock = newBlock
-    peerName = myBlock.getMetadata('name')
-    try:
-        if len(peerName) > 20:
-            raise onionrexceptions.InvalidMetdata('Peer name specified is too large')
-    except TypeError:
-        pass
-    except onionrexceptions.InvalidMetadata:
-        pass
-    else:
-        if signer in self.api.get_core().listPeers():
-            api.get_core().setPeerInfo(signer, 'name', peerName)
-            logger.info('%s is now using the name %s.' % (signer, api.get_utils().escapeAnsi(peerName)))
-
 def _processForwardKey(api, myBlock):
     '''
         Get the forward secrecy key specified by the user for us to use
@@ -67,12 +49,8 @@ def on_processblocks(api):
 
     # Process specific block types
 
-    # userInfo blocks, such as for setting username
-    if blockType == 'userInfo':
-        if api.data['validSig'] == True: # we use == True for type safety
-            _processUserInfo(api, myBlock)
     # forwardKey blocks, add a new forward secrecy key for a peer
-    elif blockType == 'forwardKey':
+    if blockType == 'forwardKey':
         if api.data['validSig'] == True:
             _processForwardKey(api, myBlock)
     # socket blocks

onionr/static-data/default-plugins/pms/main.py

@@ -74,6 +74,7 @@ class OnionrMail:
         logger.info('Decrypting messages...')
         choice = ''
         displayList = []
+        subject = ''
 
         # this could use a lot of memory if someone has recieved a lot of messages
         for blockHash in self.myCore.getBlocksByType('pm'):
@@ -97,7 +98,12 @@ class OnionrMail:
                 senderDisplay = senderKey
 
             blockDate = pmBlocks[blockHash].getDate().strftime("%m/%d %H:%M")
-            displayList.append('%s. %s - %s: %s' % (blockCount, blockDate, senderDisplay[:12], blockHash))
+            try:
+                subject = pmBlocks[blockHash].bmetadata['subject']
+            except KeyError:
+                subject = ''
+            
+            displayList.append('%s. %s - %s - <%s>: %s' % (blockCount, blockDate, senderDisplay[:12], subject[:10], blockHash))
         while choice not in ('-q', 'q', 'quit'):
             for i in displayList:
                 logger.info(i)
@@ -188,6 +194,7 @@ class OnionrMail:
     def draftMessage(self, recip=''):
         message = ''
         newLine = ''
+        subject = ''
         entering = False
         if len(recip) == 0:
             entering = True
@@ -207,21 +214,30 @@ class OnionrMail:
             else:
                 # if -q or ctrl-c/d, exit function here, otherwise we successfully got the public key
                 return
+        try:
+            subject = logger.readline('Message subject: ')
+        except (KeyboardInterrupt, EOFError):
+            pass
         
-        logger.info('Enter your message, stop by entering -q on a new line.')
+        cancelEnter = False
+        logger.info('Enter your message, stop by entering -q on a new line. -c to cancel')
         while newLine != '-q':
             try:
                 newLine = input()
             except (KeyboardInterrupt, EOFError):
-                pass
+                cancelEnter = True
+            if newLine == '-c':
+                cancelEnter = True
+                break
             if newLine == '-q':
                 continue
             newLine += '\n'
             message += newLine
 
+        if not cancelEnter:
             logger.info('Inserting encrypted message as Onionr block....')
 
-        blockID = self.myCore.insertBlock(message, header='pm', encryptType='asym', asymPeer=recip, sign=True)
+            blockID = self.myCore.insertBlock(message, header='pm', encryptType='asym', asymPeer=recip, sign=True, meta={'subject': subject})
             self.sentboxTools.addToSent(blockID, recip, message)
     def menu(self):
         choice = ''

onionr/static-data/default_config.json

@@ -1,14 +1,14 @@
 {
     "general" : {
         "dev_mode" : true,
-        "display_header" : true,
+        "display_header" : false,
-        "minimum_block_pow": 5,
+        "minimum_block_pow": 1,
-        "minimum_send_pow": 5,
+        "minimum_send_pow": 1,
         "socket_servers": false,
         "security_level": 0,
         "max_block_age": 2678400,
-        "public_key": "",
+        "bypass_tor_check": false,
-        "use_new_api_server": false
+        "public_key": ""
     },
 
     "www" : {
@@ -50,7 +50,7 @@
 
         "file": {
             "output": true,
-            "path": "data/output.log"
+            "path": "output.log"
         },
 
         "console" : {
@@ -70,7 +70,7 @@
      },
 
     "allocations" : {
-        "disk" : 10000000000,
+        "disk" : 100000000,
         "net_total" : 1000000000,
         "blockCache" : 5000000,
         "blockCacheTotal" : 50000000
@@ -79,7 +79,7 @@
     "peers" : {
         "minimum_score" : -100,
         "max_stored_peers" : 5000,
-        "max_connect" : 10
+        "max_connect" : 1000
     },
 
     "timers" : {

onionr/static-data/www/board/board.js

@@ -0,0 +1,58 @@
+webpassword = ''
+requested = []
+
+document.getElementById('webpassWindow').style.display = 'block';
+
+var windowHeight = window.innerHeight;
+document.getElementById('webpassWindow').style.height = windowHeight + "px";
+
+function httpGet(theUrl) {
+    var xmlHttp = new XMLHttpRequest()
+    xmlHttp.open( "GET", theUrl, false ) // false for synchronous request
+    xmlHttp.setRequestHeader('token', webpassword)
+    xmlHttp.send( null )
+    if (xmlHttp.status == 200){
+        return xmlHttp.responseText
+    }
+    else{
+        return "";
+    }
+}
+function appendMessages(msg){
+    el = document.createElement('div')
+    el.className = 'entry'
+    el.innerText = msg
+    document.getElementById('feed').appendChild(el)
+    document.getElementById('feed').appendChild(document.createElement('br'))
+}
+
+function getBlocks(){
+    if (document.getElementById('none') !== null){
+        document.getElementById('none').remove();
+
+    }
+    var feedText =  httpGet('/getblocksbytype/txt')
+    var blockList = feedText.split(',')
+    for (i = 0; i < blockList.length; i++){
+        if (! requested.includes(blockList[i])){
+            bl = httpGet('/gethtmlsafeblockdata/' + blockList[i])
+            appendMessages(bl)
+            requested.push(blockList[i])
+            }
+        }
+}
+
+document.getElementById('registerPassword').onclick = function(){
+    webpassword = document.getElementById('webpassword').value
+    if (httpGet('/ping') === 'pong!'){
+        document.getElementById('webpassWindow').style.display = 'none'
+        getBlocks()
+    }
+    else{
+        alert('Sorry, but that password appears invalid.')
+    }
+}
+
+document.getElementById('refreshFeed').onclick = function(){
+    getBlocks()
+}

onionr/static-data/www/board/index.html

@@ -0,0 +1,22 @@
+<!DOCTYPE HTML>
+<html>
+    <head>
+        <meta charset='utf-8'>
+        <meta name='viewport' content='width=device-width, initial-scale=1.0'>
+        <title>
+            OnionrBoard
+        </title>
+        <link rel='stylesheet' href='theme.css'>
+    </head>
+    <body>    
+        <div id='webpassWindow' class='hidden'>
+                <p>Welcome to OnionrBoard</p>
+                <p>Please enter the webpassword. You can get this from running the 'details' command in Onionr.</p>
+                <input id='webpassword' type='password' placeholder="Web password for daemon" value='CBF15ED9782FB482339E5F5B9DDCF3E58E523E71E8E9EF480596817AB5EA2E63'>
+                <button id='registerPassword'>Unlock Onionr</button>
+        </div>
+        <input type='button' id='refreshFeed' value='Refresh Feed'>
+        <div id='feed'><span id='none'>None Yet :)</span></div>
+        <script src='board.js'></script>
+    </body>
+</html>

onionr/static-data/www/board/theme.css

@@ -0,0 +1,31 @@
+h1, h2, h3{
+    font-family: sans-serif;
+}
+.hidden{
+    display: none;
+}
+p{
+    font-family: sans-serif;
+}
+#webpassWindow{
+    background-color: black;
+    border: 1px solid black;
+    border-radius: 5px;
+    width: 100%;
+    z-index: 2;
+    color: white;
+    text-align: center;
+}
+
+.entry{
+    color: red;
+}
+
+#feed{
+    margin-left: 2%;
+    margin-right: 25%;
+    margin-top: 1em;
+    border: 2px solid black;
+    padding: 5px;
+    min-height: 50px;
+}

onionr/static-data/www/mail/index.html

@@ -0,0 +1,24 @@
+<!DOCTYPE html>
+<html>
+    <head>
+        <meta charset='utf-8'>
+        <title>
+            Onionr Mail
+        </title>
+        <link rel='stylesheet' href='/shared/style/modal.css'>
+        <link rel='stylesheet' href='/shared/main/style.css'>
+        <link rel='stylesheet' href='/mail/mail.css'>
+    </head>
+    <body>
+        <div id="infoOverlay" class='overlay'>
+        </div>
+        <img class='logo' src='/shared/onionr-icon.png' alt='onionr logo'>
+        <span class='logoText'>Onionr Mail</span>
+        <div class='content'>
+            <button class='refresh'>Refresh</button>
+            <div id='threads' class='threads'></div>
+        </div>
+        <script src='/shared/misc.js'></script>
+        <script src='/mail/mail.js'></script>
+    </body>
+</html>

onionr/static-data/www/mail/mail.css

@@ -0,0 +1,7 @@
+.threads div{
+    padding-top: 1em;
+}
+.threads div span{
+    padding-left: 0.5em;
+    padding-right: 0.5em;
+}

onionr/static-data/www/mail/mail.js

@@ -0,0 +1,52 @@
+pms = ''
+threadPart = document.getElementById('threads')
+function getInbox(){
+    for(var i = 0; i < pms.length; i++) {
+        fetch('/getblockdata/' + pms[i], {
+            headers: {
+              "token": webpass
+            }})
+        .then((resp) => resp.json()) // Transform the data into json
+        .then(function(resp) {
+
+            var entry = document.createElement('div')
+
+            var bHashDisplay = document.createElement('a')
+            var senderInput = document.createElement('input')
+            var subjectLine = document.createElement('span')
+            var dateStr = document.createElement('span')
+            var humanDate = new Date(0)
+            humanDate.setUTCSeconds(resp['meta']['time'])
+            senderInput.value = resp['meta']['signer']
+            bHashDisplay.innerText = pms[i - 1].substring(0, 10)
+            bHashDisplay.setAttribute('hash', pms[i - 1]);
+            senderInput.readOnly = true
+            dateStr.innerText = humanDate.toString()
+            if (resp['metadata']['subject'] === undefined || resp['metadata']['subject'] === null) {
+                subjectLine.innerText = '()'
+            }
+            else{
+                subjectLine.innerText = '(' + resp['metadata']['subject'] + ')'
+            }
+            //entry.innerHTML = 'sender ' + resp['meta']['signer'] + ' - ' + resp['meta']['time'] 
+            threadPart.appendChild(entry)
+            entry.appendChild(bHashDisplay)
+            entry.appendChild(senderInput)
+            entry.appendChild(subjectLine)
+            entry.appendChild(dateStr)
+            
+          }.bind([pms, i]))
+    }
+
+}
+
+fetch('/getblocksbytype/pm', {
+    headers: {
+      "token": webpass
+    }})
+.then((resp) => resp.text()) // Transform the data into json
+.then(function(data) {
+    pms = data.split(',')
+    getInbox(pms)
+  })
+

onionr/static-data/www/private/index.html

@@ -0,0 +1,33 @@
+<!DOCTYPE html>
+<html>
+    <head>
+        <meta charset='utf-8'>
+        <title>
+            Onionr
+        </title>
+        <link rel='stylesheet' href='/shared/style/modal.css'>
+        <link rel='stylesheet' href='/shared/main/style.css'>
+    </head>
+    <body>
+        <div id="shutdownNotice" class='overlay'>
+            <div>
+                <p>Your node will shutdown. Thank you for using Onionr.</p>
+            </div>
+        </div>
+        <img class='logo' src='/shared/onionr-icon.png' alt='onionr logo'>
+        <span class='logoText'>Onionr Web Control Panel</span>
+        <div class='content'>
+            <button id='shutdownNode'>Shutdown Node</button> <button id='refreshStats'>Refresh Stats</button>
+            <br><br><a class='idLink' href='/mail/'>Mail</a>
+            <h2>Stats</h2>
+            <p>Uptime: <span id='uptime'></span></p>
+            <p>Stored Blocks: <span id='storedBlocks'></span></p>
+            <p>Blocks in queue: <span id='blockQueue'></span></p>
+            <p>Connected nodes:</p>
+            <pre id='connectedNodes'></pre>
+        </div>
+        <script src='/shared/misc.js'></script>
+        <script src='/shared/main/stats.js'></script>
+        <script src='/shared/panel.js'></script>
+    </body>
+</html>

onionr/static-data/www/shared/main/stats.js

@@ -0,0 +1,32 @@
+/*
+
+    Onionr - P2P Anonymous Storage Network
+
+    This file loads stats to show on the main node web page
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>
+*/
+uptimeDisplay = document.getElementById('uptime')
+connectedDisplay = document.getElementById('connectedNodes')
+storedBlockDisplay = document.getElementById('storedBlocks')
+queuedBlockDisplay = document.getElementById('blockQueue')
+
+function getStats(){
+    stats = JSON.parse(httpGet('getstats', webpass))
+    uptimeDisplay.innerText = stats['uptime'] + ' seconds'
+    connectedDisplay.innerText = stats['connectedNodes']
+    storedBlockDisplay.innerText = stats['blockCount']
+    queuedBlockDisplay.innerText = stats['blockQueueCount']
+}
+getStats()

onionr/static-data/www/shared/main/style.css

@@ -0,0 +1,140 @@
+body{
+    background-color: #2c2b3f;
+    color: white;
+}
+
+a, a:visited{
+    color: white;
+}
+.center{
+    text-align: center;
+}
+footer{
+    margin-top: 2em;
+    margin-bottom: 0.5em;
+}
+
+body{
+    margin-left: 3em;
+    padding: 1em;
+}
+.onionrMenu{
+    max-width: 25%;
+    margin-left: 2%;
+    margin-right: 10%;
+    font-family: sans-serif;
+}
+.onionrMenu li{
+    list-style-type: none;
+    margin-top: 3px;
+    font-size: 125%;
+}
+.onionrMenu li:hover{
+    color: red;
+}
+.box {
+    display: flex;
+    align-items:center;
+ }
+ .logo{
+    max-width: 25%;
+    vertical-align: middle;
+ }
+ .logoText{
+     font-family: sans-serif;
+     font-size: 2em;
+     margin-top: 1em;
+     margin-left: 1%;
+ }
+ .main{
+     min-height: 500px;
+ }
+
+.content{
+    margin-top: 3em;
+    margin-left: 0%;
+    margin-right: 40%;
+    background-color: white;
+    color: black;
+    padding-right: 5%;
+    padding-left: 3%;
+    padding-bottom: 2em;
+    padding-top: 0.5em;
+    border: 1px solid black;
+    border-radius: 10px;
+    min-height: 300px;
+}
+.content p{
+    text-align: justify;
+}
+.content img{
+    max-width: 35%;
+}
+.content a, .content a:visited{
+    color: black;
+}
+
+.stats{
+    margin-top: 1em;
+    background-color: #0c1049;
+    padding: 5px;
+    margin-right: 45%;
+    font-family: sans-serif;
+}
+.statDesc{
+    background-color: black;
+    padding: 5px;
+    margin-right: 1%;
+    margin-left: -5px;
+}
+
+.stats noscript{
+    color: blue;
+}
+
+.statItem{
+    padding-left: 10px;
+    float: right;
+    margin-right: 5px;
+}
+
+.warn{
+    color: orangered;
+}
+
+@media only screen and (max-width: 640px) {
+    .onionrMenu{
+        margin-left: 0%;
+    }
+    body{
+        margin-left: 0em;
+    }
+    .content{
+        margin-left: 1%;
+        margin-right: 2%;
+    }
+    .content img{
+        max-width: 85%;
+    }
+    .stats{
+        margin-right: 1%;
+    }
+    .statItem{
+        float: initial;
+        display: block;
+    }
+}
+
+/*https://stackoverflow.com/a/16778646*/
+.overlay {
+    visibility: hidden;
+    position: absolute;
+    left: 0px;
+    top: 0px;
+    width:100%;
+    opacity: 0.9;
+    height:100%;
+    text-align:center;
+    z-index: 1000;
+    background-color: black;
+ }

onionr/static-data/www/shared/misc.js

@@ -0,0 +1,44 @@
+webpass = document.location.hash.replace('#', '')
+nowebpass = false
+if (typeof webpass == "undefined"){
+    webpass = localStorage['webpass']
+}
+else{
+    localStorage['webpass'] = webpass
+    //document.location.hash = ''
+}
+if (typeof webpass == "undefined" || webpass == ""){
+    alert('Web password was not found in memory or URL')
+    nowebpass = true
+}
+
+function httpGet(theUrl) {
+    var xmlHttp = new XMLHttpRequest()
+    xmlHttp.open( "GET", theUrl, false ) // false for synchronous request
+    xmlHttp.setRequestHeader('token', webpass)
+    xmlHttp.send( null )
+    if (xmlHttp.status == 200){
+        return xmlHttp.responseText
+    }
+    else{
+        return ""
+    }
+}
+function overlay(overlayID) {
+    el = document.getElementById(overlayID)
+   el.style.visibility = (el.style.visibility == "visible") ? "hidden" : "visible"
+ }
+
+var passLinks = document.getElementsByClassName("idLink")
+ for(var i = 0; i < passLinks.length; i++) {
+    passLinks[i].href += '#' + webpass
+ }
+ 
+var refreshLinks = document.getElementsByClassName("refresh")
+
+for(var i = 0; i < refreshLinks.length; i++) {
+    //Can't use .reload because of webpass
+    refreshLinks[i].onclick = function(){
+        location.reload()
+    }
+}

onionr/static-data/www/shared/onionrblocks.js

@@ -0,0 +1,7 @@
+class Block {
+    constructor(hash, raw) {
+        this.hash = hash;
+        this.raw = raw;
+    }
+}
+   

onionr/static-data/www/shared/panel.js

@@ -0,0 +1,12 @@
+shutdownBtn = document.getElementById('shutdownNode')
+refreshStatsBtn = document.getElementById('refreshStats')
+shutdownBtn.onclick = function(){
+    if (! nowebpass){
+        httpGet('shutdownclean')
+        overlay('shutdownNotice')
+    }
+}
+
+refreshStatsBtn.onclick = function(){
+    getStats()
+}

onionr/static-data/www/ui/dist/js/main.js

@@ -704,7 +704,7 @@ if(tt !== null && tt !== undefined) {
 if(getWebPassword() === null) {
     var password = "";
     while(password.length != 64) {
-        password = prompt("Please enter the web password (run `./RUN-LINUX.sh --get-password`)");
+        password = prompt("Please enter the web password (run `./RUN-LINUX.sh --details`)");
     }
 
     setWebPassword(password);

onionr/storagecounter.py

@@ -42,8 +42,15 @@ class StorageCounter:
                 retData = int(dataFile.read())
         except FileNotFoundError:
             pass
+        except ValueError:
+            pass # Possibly happens when the file is empty
         return retData
     
+    def getPercent(self):
+        '''Return percent (decimal/float) of disk space we're using'''
+        amount = self.getAmount()
+        return round(amount / self._core.config.get('allocations.disk'), 2)
+
     def addBytes(self, amount):
         '''Record that we are now using more disk space, unless doing so would exceed configured max'''
         newAmount = amount + self.getAmount()

start-daemon.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/bash
+cd "$(dirname "$0")"
+echo "starting Onionr daemon..."
+echo "run onionr.sh stop to stop the daemon, or onionr.sh start to get output"
+nohup ./onionr.sh start & disown > /dev/null