Merge branch 'dbstorage' of gitlab.com:beardog/Onionr into dbstorage

.gitlab-ci.yml

@@ -1,6 +0,0 @@
-test:
-  script:
-  - apt-get update -qy
-  - apt-get install -y python3-dev python3-pip tor
-  - pip3 install -r requirements.txt
-  - make test

.travis.yml

@@ -1,8 +0,0 @@
-language: python
-python:
-        - "3.6.4"
-# install dependencies
-install:
-        - sudo apt install tor
-        - pip install -r requirements.txt
-script: make test

onionr-daemon-linux

@@ -0,0 +1,2 @@
+#!/usr/bin/sh
+nohup ./run-linux start & disown

onionr/api.py

@@ -17,7 +17,7 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import flask
+import flask, cgi
 from flask import request, Response, abort, send_from_directory
 from gevent.pywsgi import WSGIServer
 import sys, random, threading, hmac, hashlib, base64, time, math, os, json
@@ -221,7 +221,7 @@ class API:
             This initilization defines all of the API entry points and handlers for the endpoints and errors
             This also saves the used host (random localhost IP address) to the data folder in host.txt
         '''
-
+        # assert isinstance(onionrInst, onionr.Onionr)
         # configure logger and stuff
         onionr.Onionr.setupConfig('data/', self = self)
 
@@ -234,6 +234,8 @@ class API:
         bindPort = int(config.get('client.client.port', 59496))
         self.bindPort = bindPort
 
+        self.whitelistEndpoints = ('site', 'www', 'onionrhome', 'board', 'boardContent', 'sharedContent')
+
         self.clientToken = config.get('client.webpassword')
         self.timeBypassToken = base64.b16encode(os.urandom(32)).decode()
 
@@ -249,6 +251,8 @@ class API:
             '''Validate request has set password and is the correct hostname'''
             if request.host != '%s:%s' % (self.host, self.bindPort):
                 abort(403)
+            if request.endpoint in self.whitelistEndpoints:
+                return
             try:
                 if not hmac.compare_digest(request.headers['token'], self.clientToken):
                     abort(403)
@@ -257,7 +261,8 @@ class API:
 
         @app.after_request
         def afterReq(resp):
-            resp.headers["Content-Security-Policy"] =  "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'"
+            #resp.headers["Content-Security-Policy"] =  "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'"
+            resp.headers['Content-Security-Policy'] = "default-src 'none'; script-src 'self'; object-src 'none'; style-src 'self'; img-src 'self'; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'self'"
             resp.headers['X-Frame-Options'] = 'deny'
             resp.headers['X-Content-Type-Options'] = "nosniff"
             resp.headers['X-API'] = onionr.API_VERSION
@@ -265,20 +270,57 @@ class API:
             resp.headers['Date'] = 'Thu, 1 Jan 1970 00:00:00 GMT' # Clock info is probably useful to attackers. Set to unix epoch.
             return resp
 
+        @app.route('/board/', endpoint='board')
+        def loadBoard():
+            return send_from_directory('static-data/www/board/', "index.html")
+
+        @app.route('/board/<path:path>', endpoint='boardContent')
+        def boardContent(path):
+            return send_from_directory('static-data/www/board/', path)
+        @app.route('/shared/<path:path>', endpoint='sharedContent')
+        def sharedContent(path):
+            return send_from_directory('static-data/www/shared/', path)
+
+        @app.route('/www/<path:path>', endpoint='www')
+        def wwwPublic(path):
+            if not config.get("www.private.run", True):
+                abort(403)
+            return send_from_directory(config.get('www.private.path', 'static-data/www/private/'), path)
+
         @app.route('/ping')
         def ping():
             return Response("pong!")
 
-        @app.route('/')
+        @app.route('/', endpoint='onionrhome')
         def hello():
-            return Response("hello client")
+            return Response("Welcome to Onionr")
+        
+        @app.route('/getblocksbytype/<name>')
+        def getBlocksByType(name):
+            blocks = self._core.getBlocksByType(name)
+            return Response(','.join(blocks))
+        
+        @app.route('/gethtmlsafeblockdata/<name>')
+        def getData(name):
+            resp = ''
+            if self._core._utils.validateHash(name):
+                try:
+                    resp =  cgi.escape(Block(name).bcontent, quote=True)
+                except TypeError:
+                    pass
+            else:
+                abort(404)
+            return Response(resp)
 
-        @app.route('/site/<name>')
+        @app.route('/site/<name>', endpoint='site')
-        def site():
+        def site(name):
-            bHash = block
+            bHash = name
             resp = 'Not Found'
             if self._core._utils.validateHash(bHash):
-                resp = Block(bHash).bcontent
+                try:
+                    resp = Block(bHash).bcontent
+                except TypeError:
+                    pass
                 try:
                     resp = base64.b64decode(resp)
                 except:

onionr/communicator2.py

@@ -102,7 +102,7 @@ class OnionrCommunicatorDaemon:
         OnionrCommunicatorTimers(self, self.daemonTools.cooldownPeer, 30, requiresPeer=True)
         OnionrCommunicatorTimers(self, self.uploadBlock, 10, requiresPeer=True, maxThreads=1)
         OnionrCommunicatorTimers(self, self.daemonCommands, 6, maxThreads=1)
-        OnionrCommunicatorTimers(self, self.detectAPICrash, 5, maxThreads=1)
+        OnionrCommunicatorTimers(self, self.detectAPICrash, 30, maxThreads=1)
         deniableBlockTimer = OnionrCommunicatorTimers(self, self.daemonTools.insertDeniableBlock, 180, requiresPeer=True, maxThreads=1)
 
         netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600)

onionr/core.py

@@ -681,7 +681,10 @@ class Core:
             Inserts a block into the network
             encryptType must be specified to encrypt a block
         '''
-
+        allocationReachedMessage = 'Cannot insert block, disk allocation reached.'
+        if self._utils.storageCounter.isFull():
+            logger.error(allocationReachedMessage)
+            return False
         retData = False
         # check nonce
         dataNonce = self._utils.bytesToStr(self._crypto.sha3Hash(data))
@@ -775,13 +778,18 @@ class Core:
         proof = onionrproofs.POW(metadata, data)
         payload = proof.waitForResult()
         if payload != False:
-            retData = self.setData(payload)
+            try:
-            # Tell the api server through localCommand to wait for the daemon to upload this block to make stastical analysis more difficult
+                retData = self.setData(payload)
-            self._utils.localCommand('waitforshare/' + retData)
+            except onionrexceptions.DiskAllocationReached:
-            self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
+                logger.error(allocationReachedMessage)
-            #self.setBlockType(retData, meta['type'])
+                retData = False
-            self._utils.processBlockMetadata(retData)
+            else:
-            self.daemonQueueAdd('uploadBlock', retData)
+                # Tell the api server through localCommand to wait for the daemon to upload this block to make stastical analysis more difficult
+                self._utils.localCommand('waitforshare/' + retData)
+                self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
+                #self.setBlockType(retData, meta['type'])
+                self._utils.processBlockMetadata(retData)
+                self.daemonQueueAdd('uploadBlock', retData)
 
         if retData != False:
             events.event('insertBlock', onionr = None, threaded = False)

onionr/onionr.py

@@ -351,46 +351,9 @@ class Onionr:
                 except IndexError:
                     logger.error('Friend ID is required.')
                 except onionrexceptions.KeyNotKnown:
-                    logger.error('That peer is not in our database')
+                    self.onionrCore.addPeer(friend)
-                else:
-                    if action == 'add':
-                        friend.setTrust(1)
-                        logger.info('Added %s as friend.' % (friend.publicKey,))
-                    else:
-                        friend.setTrust(0)
-                        logger.info('Removed %s as friend.' % (friend.publicKey,))
-            else:
-                logger.info('Syntax: friend add/remove/list [address]')
-
-
-    def friendCmd(self):
-        '''List, add, or remove friend(s)
-        Changes their peer DB entry.
-        '''
-        friend = ''
-        try:
-            # Get the friend command
-            action = sys.argv[2]
-        except IndexError:
-            logger.info('Syntax: friend add/remove/list [address]')
-        else:
-            action = action.lower()
-            if action == 'list':
-                # List out peers marked as our friend
-                for friend in self.onionrCore.listPeers(randomOrder=False, trust=1):
-                    if friend == self.onionrCore._crypto.pubKey: # do not list our key
-                        continue
-                    friendProfile = onionrusers.OnionrUser(self.onionrCore, friend)
-                    logger.info(friend + ' - ' + friendProfile.getName())
-            elif action in ('add', 'remove'):
-                try:
-                    friend = sys.argv[3]
-                    if not self.onionrUtils.validatePubKey(friend):
-                        raise onionrexceptions.InvalidPubkey('Public key is invalid')
                     friend = onionrusers.OnionrUser(self.onionrCore, friend)
-                except IndexError:
+                finally:
-                    logger.error('Friend ID is required.')
-                else:
                     if action == 'add':
                         friend.setTrust(1)
                         logger.info('Added %s as friend.' % (friend.publicKey,))
@@ -400,7 +363,6 @@ class Onionr:
             else:
                 logger.info('Syntax: friend add/remove/list [address]')
 
-
     def deleteRunFiles(self):
         try:
             os.remove(self.onionrCore.publicApiHostFile)
@@ -719,7 +681,6 @@ class Onionr:
         '''
             Starts the Onionr communication daemon
         '''
-
         communicatorDaemon = './communicator2.py'
 
         # remove runcheck if it exists

onionr/onionrblockapi.py

@@ -245,8 +245,8 @@ class Block:
                         blockFile.write(self.getRaw().encode())
                 else:
                     self.hash = self.getCore().insertBlock(self.getContent(), header = self.getType(), sign = sign, meta = self.getMetadata(), expire = self.getExpire())
-                
+                if self.hash != False:
-                self.update()
+                    self.update()
 
                 return self.getHash()
             else:

onionr/onionrcrypto.py

@@ -268,8 +268,9 @@ class OnionrCrypto:
             blockHash = blockHash.decode() # bytes on some versions for some reason
         except AttributeError:
             pass
-
+        
-        difficulty = math.floor(dataLen / 1000000)
+        difficulty = onionrproofs.getDifficultyForNewBlock(blockContent, ourBlock=False)
+        
         if difficulty < int(config.get('general.minimum_block_pow')):
             difficulty = int(config.get('general.minimum_block_pow'))
         mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode()

onionr/onionrproofs.py

@@ -19,7 +19,57 @@
 '''
 
 import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, logger, sys, base64, json
-import core, config
+import core, onionrutils, config
+import onionrblockapi
+
+def getDifficultyModifier(coreOrUtilsInst=None):
+    '''Accepts a core or utils instance returns 
+    the difficulty modifier for block storage based 
+    on a variety of factors, currently only disk use.
+    '''
+    classInst = coreOrUtilsInst
+    retData = 0
+    if isinstance(classInst, core.Core):
+        useFunc = classInst._utils.storageCounter.getPercent
+    elif isinstance(classInst, onionrutils.OnionrUtils):
+        useFunc = classInst.storageCounter.getPercent
+    else:
+        useFunc = core.Core()._utils.storageCounter.getPercent
+
+    percentUse = useFunc()
+
+    if percentUse >= 0.50:
+        retData += 1
+    elif percentUse >= 0.75:
+            retData += 2
+    elif percentUse >= 0.95:
+            retData += 3
+
+    return retData
+
+def getDifficultyForNewBlock(data, ourBlock=True):
+    '''
+    Get difficulty for block. Accepts size in integer, Block instance, or str/bytes full block contents
+    '''
+    retData = 0
+    dataSize = 0
+    if isinstance(data, onionrblockapi.Block):
+        dataSize = len(data.getRaw().encode('utf-8'))
+    elif isinstance(data, str):
+        dataSize = len(data.encode('utf-8'))
+    elif isinstance(data, bytes):
+        dataSize = len(data)
+    elif isinstance(data, int):
+        dataSize = data
+    else:
+        raise ValueError('not Block, str, or int')
+    if ourBlock:
+        minDifficulty = config.get('general.minimum_send_pow')
+    else:
+        minDifficulty = config.get('general.minimum_block_pow')
+
+    retData = max(minDifficulty, math.floor(dataSize / 1000000)) + getDifficultyModifier()
+    return retData
 
 def getHashDifficulty(h):
     '''
@@ -55,6 +105,7 @@ class DataPOW:
         self.difficulty = 0
         self.data = data
         self.threadCount = threadCount
+        self.rounds = 0
         config.reload()
 
         if forceDifficulty == 0:
@@ -96,6 +147,7 @@ class DataPOW:
         while self.hashing:
             rand = nacl.utils.random()
             token = nacl.hash.blake2b(rand + self.data).decode()
+            self.rounds += 1
             #print(token)
             if self.puzzle == token[0:self.difficulty]:
                 self.hashing = False
@@ -106,6 +158,7 @@ class DataPOW:
             endTime = math.floor(time.time())
             if self.reporting:
                 logger.debug('Found token after %s seconds: %s' % (endTime - startTime, token), timestamp=True)
+                logger.debug('Round count: %s' % (self.rounds,))
             self.result = (token, rand)
 
     def shutdown(self):
@@ -146,18 +199,28 @@ class DataPOW:
         return result
 
 class POW:
-    def __init__(self, metadata, data, threadCount = 5):
+    def __init__(self, metadata, data, threadCount = 5, forceDifficulty=0, coreInst=None):
         self.foundHash = False
         self.difficulty = 0
         self.data = data
         self.metadata = metadata
         self.threadCount = threadCount
 
-        dataLen = len(data) + len(json.dumps(metadata))
+        try:
-        self.difficulty = math.floor(dataLen / 1000000)
+            assert isinstance(coreInst, core.Core)
-        if self.difficulty <= 2:
+        except AssertionError:
-            self.difficulty = int(config.get('general.minimum_block_pow'))
+            myCore = core.Core()
+        else:
+            myCore = coreInst
 
+        dataLen = len(data) + len(json.dumps(metadata))
+
+        if forceDifficulty > 0:
+            self.difficulty = forceDifficulty
+        else:
+            # Calculate difficulty. Dumb for now, may use good algorithm in the future.
+            self.difficulty = getDifficultyForNewBlock(dataLen)
+            
         try:
             self.data = self.data.encode()
         except AttributeError:
@@ -167,8 +230,7 @@ class POW:
 
         self.mainHash = '0' * 64
         self.puzzle = self.mainHash[0:min(self.difficulty, len(self.mainHash))]
-        
+
-        myCore = core.Core()
         for i in range(max(1, threadCount)):
             t = threading.Thread(name = 'thread%s' % i, target = self.pow, args = (True,myCore))
             t.start()

onionr/onionrstorage.py

@@ -0,0 +1,31 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This file handles block storage, providing an abstraction for storing blocks between file system and database
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import core
+class OnionrStorage:
+    def __init__(self, coreInst):
+        assert isinstance(coreInst, core.Core)
+        self._core = coreInst
+        return
+    
+    def store(self, hash, data):
+        return
+    
+    def getData(self, hash):
+        return

onionr/onionrutils.py

@@ -155,18 +155,21 @@ class OnionrUtils:
         '''
             Send a command to the local http API server, securely. Intended for local clients, DO NOT USE for remote peers.
         '''
-
         config.reload()
         self.getTimeBypassToken()
         # TODO: URL encode parameters, just as an extra measure. May not be needed, but should be added regardless.
         hostname = ''
+        maxWait = 5
+        waited = 0
         while hostname == '':
             try:
                 with open(self._core.privateApiHostFile, 'r') as host:
                     hostname = host.read()
             except FileNotFoundError:
-                print('wat')
                 time.sleep(1)
+                waited += 1
+                if waited == maxWait:
+                    return False
         if data != '':
             data = '&data=' + urllib.parse.quote_plus(data)
         payload = 'http://%s:%s/%s%s' % (hostname, config.get('client.client.port'), command, data)

onionr/static-data/default_config.json

@@ -1,14 +1,13 @@
 {
     "general" : {
         "dev_mode" : true,
-        "display_header" : true,
+        "display_header" : false,
-        "minimum_block_pow": 5,
+        "minimum_block_pow": 3,
-        "minimum_send_pow": 5,
+        "minimum_send_pow": 3,
         "socket_servers": false,
         "security_level": 0,
         "max_block_age": 2678400,
-        "public_key": "",
+        "public_key": ""
-        "use_new_api_server": false
     },
 
     "www" : {
@@ -70,7 +69,7 @@
      },
 
     "allocations" : {
-        "disk" : 10000000000,
+        "disk" : 100000000,
         "net_total" : 1000000000,
         "blockCache" : 5000000,
         "blockCacheTotal" : 50000000

onionr/static-data/www/board/board.js

@@ -0,0 +1,58 @@
+webpassword = ''
+requested = []
+
+document.getElementById('webpassWindow').style.display = 'block';
+
+var windowHeight = window.innerHeight;
+document.getElementById('webpassWindow').style.height = windowHeight + "px";
+
+function httpGet(theUrl) {
+    var xmlHttp = new XMLHttpRequest()
+    xmlHttp.open( "GET", theUrl, false ) // false for synchronous request
+    xmlHttp.setRequestHeader('token', webpassword)
+    xmlHttp.send( null )
+    if (xmlHttp.status == 200){
+        return xmlHttp.responseText
+    }
+    else{
+        return "";
+    }
+}
+function appendMessages(msg){
+    el = document.createElement('div')
+    el.className = 'entry'
+    el.innerText = msg
+    document.getElementById('feed').appendChild(el)
+    document.getElementById('feed').appendChild(document.createElement('br'))
+}
+
+function getBlocks(){
+    if (document.getElementById('none') !== null){
+        document.getElementById('none').remove();
+
+    }
+    var feedText =  httpGet('/getblocksbytype/txt')
+    var blockList = feedText.split(',')
+    for (i = 0; i < blockList.length; i++){
+        if (! requested.includes(blockList[i])){
+            bl = httpGet('/gethtmlsafeblockdata/' + blockList[i])
+            appendMessages(bl)
+            requested.push(blockList[i])
+            }
+        }
+}
+
+document.getElementById('registerPassword').onclick = function(){
+    webpassword = document.getElementById('webpassword').value
+    if (httpGet('/ping') === 'pong!'){
+        document.getElementById('webpassWindow').style.display = 'none'
+        getBlocks()
+    }
+    else{
+        alert('Sorry, but that password appears invalid.')
+    }
+}
+
+document.getElementById('refreshFeed').onclick = function(){
+    getBlocks()
+}

onionr/static-data/www/board/index.html

@@ -0,0 +1,21 @@
+<!DOCTYPE HTML>
+<html>
+    <head>
+        <meta charset='utf-8'>
+        <title>
+            OnionrBoard
+        </title>
+        <link rel='stylesheet' href='theme.css'>
+    </head>
+    <body>    
+        <div id='webpassWindow' class='hidden'>
+                <p>Welcome to OnionrBoard</p>
+                <p>Please enter the webpassword. You can get this from running the 'details' command in Onionr.</p>
+                <input id='webpassword' type='password' placeholder="Web password for daemon" value='7AF13568657CE63D6DB7E686BF05537D36598ED739B21E3F023E3FD3DEA2FC8F'>
+                <button id='registerPassword'>Unlock Onionr</button>
+        </div>
+        <input type='button' id='refreshFeed' value='Refresh Feed'>
+        <div id='feed'><span id='none'>None Yet :)</span></div>
+        <script src='board.js'></script>
+    </body>
+</html>

onionr/static-data/www/board/theme.css

@@ -0,0 +1,31 @@
+h1, h2, h3{
+    font-family: sans-serif;
+}
+.hidden{
+    display: none;
+}
+p{
+    font-family: sans-serif;
+}
+#webpassWindow{
+    background-color: black;
+    border: 1px solid black;
+    border-radius: 5px;
+    width: 100%;
+    z-index: 2;
+    color: white;
+    text-align: center;
+}
+
+.entry{
+    color: red;
+}
+
+#feed{
+    margin-left: 2%;
+    margin-right: 25%;
+    margin-top: 1em;
+    border: 2px solid black;
+    padding: 5px;
+    min-height: 50px;
+}

onionr/storagecounter.py

@@ -43,6 +43,11 @@ class StorageCounter:
         except FileNotFoundError:
             pass
         return retData
+    
+    def getPercent(self):
+        '''Return percent (decimal/float) of disk space we're using'''
+        amount = self.getAmount()
+        return round(amount / self._core.config.get('allocations.disk'), 2)
 
     def addBytes(self, amount):
         '''Record that we are now using more disk space, unless doing so would exceed configured max'''