diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 292dfb14..00000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,6 +0,0 @@ -test: - script: - - apt-get update -qy - - apt-get install -y python3-dev python3-pip tor - - pip3 install -r requirements.txt - - make test \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 603021b5..00000000 --- a/.travis.yml +++ /dev/null @@ -1,8 +0,0 @@ -language: python -python: - - "3.6.4" -# install dependencies -install: - - sudo apt install tor - - pip install -r requirements.txt -script: make test diff --git a/onionr-daemon-linux b/onionr-daemon-linux new file mode 100644 index 00000000..d72ac015 --- /dev/null +++ b/onionr-daemon-linux @@ -0,0 +1,2 @@ +#!/usr/bin/sh +nohup ./run-linux start & disown diff --git a/onionr/api.py b/onionr/api.py index f983677d..75f02a13 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import flask +import flask, cgi from flask import request, Response, abort, send_from_directory from gevent.pywsgi import WSGIServer import sys, random, threading, hmac, hashlib, base64, time, math, os, json @@ -221,7 +221,7 @@ class API: This initilization defines all of the API entry points and handlers for the endpoints and errors This also saves the used host (random localhost IP address) to the data folder in host.txt ''' - + # assert isinstance(onionrInst, onionr.Onionr) # configure logger and stuff onionr.Onionr.setupConfig('data/', self = self) @@ -234,6 +234,8 @@ class API: bindPort = int(config.get('client.client.port', 59496)) self.bindPort = bindPort + self.whitelistEndpoints = ('site', 'www', 'onionrhome', 'board', 'boardContent', 'sharedContent') + self.clientToken = config.get('client.webpassword') self.timeBypassToken = base64.b16encode(os.urandom(32)).decode() @@ -249,6 +251,8 @@ class API: '''Validate request has set password and is the correct hostname''' if request.host != '%s:%s' % (self.host, self.bindPort): abort(403) + if request.endpoint in self.whitelistEndpoints: + return try: if not hmac.compare_digest(request.headers['token'], self.clientToken): abort(403) @@ -257,7 +261,8 @@ class API: @app.after_request def afterReq(resp): - resp.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'" + #resp.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'" + resp.headers['Content-Security-Policy'] = "default-src 'none'; script-src 'self'; object-src 'none'; style-src 'self'; img-src 'self'; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'self'" resp.headers['X-Frame-Options'] = 'deny' resp.headers['X-Content-Type-Options'] = "nosniff" resp.headers['X-API'] = onionr.API_VERSION @@ -265,20 +270,57 @@ class API: resp.headers['Date'] = 'Thu, 1 Jan 1970 00:00:00 GMT' # Clock info is probably useful to attackers. Set to unix epoch. return resp + @app.route('/board/', endpoint='board') + def loadBoard(): + return send_from_directory('static-data/www/board/', "index.html") + + @app.route('/board/', endpoint='boardContent') + def boardContent(path): + return send_from_directory('static-data/www/board/', path) + @app.route('/shared/', endpoint='sharedContent') + def sharedContent(path): + return send_from_directory('static-data/www/shared/', path) + + @app.route('/www/', endpoint='www') + def wwwPublic(path): + if not config.get("www.private.run", True): + abort(403) + return send_from_directory(config.get('www.private.path', 'static-data/www/private/'), path) + @app.route('/ping') def ping(): return Response("pong!") - @app.route('/') + @app.route('/', endpoint='onionrhome') def hello(): - return Response("hello client") + return Response("Welcome to Onionr") + + @app.route('/getblocksbytype/') + def getBlocksByType(name): + blocks = self._core.getBlocksByType(name) + return Response(','.join(blocks)) + + @app.route('/gethtmlsafeblockdata/') + def getData(name): + resp = '' + if self._core._utils.validateHash(name): + try: + resp = cgi.escape(Block(name).bcontent, quote=True) + except TypeError: + pass + else: + abort(404) + return Response(resp) - @app.route('/site/') - def site(): - bHash = block + @app.route('/site/', endpoint='site') + def site(name): + bHash = name resp = 'Not Found' if self._core._utils.validateHash(bHash): - resp = Block(bHash).bcontent + try: + resp = Block(bHash).bcontent + except TypeError: + pass try: resp = base64.b64decode(resp) except: diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 5d045a69..8e10e8b1 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -102,7 +102,7 @@ class OnionrCommunicatorDaemon: OnionrCommunicatorTimers(self, self.daemonTools.cooldownPeer, 30, requiresPeer=True) OnionrCommunicatorTimers(self, self.uploadBlock, 10, requiresPeer=True, maxThreads=1) OnionrCommunicatorTimers(self, self.daemonCommands, 6, maxThreads=1) - OnionrCommunicatorTimers(self, self.detectAPICrash, 5, maxThreads=1) + OnionrCommunicatorTimers(self, self.detectAPICrash, 30, maxThreads=1) deniableBlockTimer = OnionrCommunicatorTimers(self, self.daemonTools.insertDeniableBlock, 180, requiresPeer=True, maxThreads=1) netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600) diff --git a/onionr/core.py b/onionr/core.py index e52c29e7..92ffad2d 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -681,7 +681,10 @@ class Core: Inserts a block into the network encryptType must be specified to encrypt a block ''' - + allocationReachedMessage = 'Cannot insert block, disk allocation reached.' + if self._utils.storageCounter.isFull(): + logger.error(allocationReachedMessage) + return False retData = False # check nonce dataNonce = self._utils.bytesToStr(self._crypto.sha3Hash(data)) @@ -775,13 +778,18 @@ class Core: proof = onionrproofs.POW(metadata, data) payload = proof.waitForResult() if payload != False: - retData = self.setData(payload) - # Tell the api server through localCommand to wait for the daemon to upload this block to make stastical analysis more difficult - self._utils.localCommand('waitforshare/' + retData) - self.addToBlockDB(retData, selfInsert=True, dataSaved=True) - #self.setBlockType(retData, meta['type']) - self._utils.processBlockMetadata(retData) - self.daemonQueueAdd('uploadBlock', retData) + try: + retData = self.setData(payload) + except onionrexceptions.DiskAllocationReached: + logger.error(allocationReachedMessage) + retData = False + else: + # Tell the api server through localCommand to wait for the daemon to upload this block to make stastical analysis more difficult + self._utils.localCommand('waitforshare/' + retData) + self.addToBlockDB(retData, selfInsert=True, dataSaved=True) + #self.setBlockType(retData, meta['type']) + self._utils.processBlockMetadata(retData) + self.daemonQueueAdd('uploadBlock', retData) if retData != False: events.event('insertBlock', onionr = None, threaded = False) diff --git a/onionr/onionr.py b/onionr/onionr.py index aadceb56..04bee60a 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -351,46 +351,9 @@ class Onionr: except IndexError: logger.error('Friend ID is required.') except onionrexceptions.KeyNotKnown: - logger.error('That peer is not in our database') - else: - if action == 'add': - friend.setTrust(1) - logger.info('Added %s as friend.' % (friend.publicKey,)) - else: - friend.setTrust(0) - logger.info('Removed %s as friend.' % (friend.publicKey,)) - else: - logger.info('Syntax: friend add/remove/list [address]') - - - def friendCmd(self): - '''List, add, or remove friend(s) - Changes their peer DB entry. - ''' - friend = '' - try: - # Get the friend command - action = sys.argv[2] - except IndexError: - logger.info('Syntax: friend add/remove/list [address]') - else: - action = action.lower() - if action == 'list': - # List out peers marked as our friend - for friend in self.onionrCore.listPeers(randomOrder=False, trust=1): - if friend == self.onionrCore._crypto.pubKey: # do not list our key - continue - friendProfile = onionrusers.OnionrUser(self.onionrCore, friend) - logger.info(friend + ' - ' + friendProfile.getName()) - elif action in ('add', 'remove'): - try: - friend = sys.argv[3] - if not self.onionrUtils.validatePubKey(friend): - raise onionrexceptions.InvalidPubkey('Public key is invalid') + self.onionrCore.addPeer(friend) friend = onionrusers.OnionrUser(self.onionrCore, friend) - except IndexError: - logger.error('Friend ID is required.') - else: + finally: if action == 'add': friend.setTrust(1) logger.info('Added %s as friend.' % (friend.publicKey,)) @@ -400,7 +363,6 @@ class Onionr: else: logger.info('Syntax: friend add/remove/list [address]') - def deleteRunFiles(self): try: os.remove(self.onionrCore.publicApiHostFile) @@ -719,7 +681,6 @@ class Onionr: ''' Starts the Onionr communication daemon ''' - communicatorDaemon = './communicator2.py' # remove runcheck if it exists diff --git a/onionr/onionrblockapi.py b/onionr/onionrblockapi.py index e6506faf..93bc3201 100644 --- a/onionr/onionrblockapi.py +++ b/onionr/onionrblockapi.py @@ -245,8 +245,8 @@ class Block: blockFile.write(self.getRaw().encode()) else: self.hash = self.getCore().insertBlock(self.getContent(), header = self.getType(), sign = sign, meta = self.getMetadata(), expire = self.getExpire()) - - self.update() + if self.hash != False: + self.update() return self.getHash() else: diff --git a/onionr/onionrcrypto.py b/onionr/onionrcrypto.py index 04c821cd..03da9213 100644 --- a/onionr/onionrcrypto.py +++ b/onionr/onionrcrypto.py @@ -268,8 +268,9 @@ class OnionrCrypto: blockHash = blockHash.decode() # bytes on some versions for some reason except AttributeError: pass - - difficulty = math.floor(dataLen / 1000000) + + difficulty = onionrproofs.getDifficultyForNewBlock(blockContent, ourBlock=False) + if difficulty < int(config.get('general.minimum_block_pow')): difficulty = int(config.get('general.minimum_block_pow')) mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode() diff --git a/onionr/onionrproofs.py b/onionr/onionrproofs.py index 9665a4cb..f1645a49 100644 --- a/onionr/onionrproofs.py +++ b/onionr/onionrproofs.py @@ -19,7 +19,57 @@ ''' import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, logger, sys, base64, json -import core, config +import core, onionrutils, config +import onionrblockapi + +def getDifficultyModifier(coreOrUtilsInst=None): + '''Accepts a core or utils instance returns + the difficulty modifier for block storage based + on a variety of factors, currently only disk use. + ''' + classInst = coreOrUtilsInst + retData = 0 + if isinstance(classInst, core.Core): + useFunc = classInst._utils.storageCounter.getPercent + elif isinstance(classInst, onionrutils.OnionrUtils): + useFunc = classInst.storageCounter.getPercent + else: + useFunc = core.Core()._utils.storageCounter.getPercent + + percentUse = useFunc() + + if percentUse >= 0.50: + retData += 1 + elif percentUse >= 0.75: + retData += 2 + elif percentUse >= 0.95: + retData += 3 + + return retData + +def getDifficultyForNewBlock(data, ourBlock=True): + ''' + Get difficulty for block. Accepts size in integer, Block instance, or str/bytes full block contents + ''' + retData = 0 + dataSize = 0 + if isinstance(data, onionrblockapi.Block): + dataSize = len(data.getRaw().encode('utf-8')) + elif isinstance(data, str): + dataSize = len(data.encode('utf-8')) + elif isinstance(data, bytes): + dataSize = len(data) + elif isinstance(data, int): + dataSize = data + else: + raise ValueError('not Block, str, or int') + if ourBlock: + minDifficulty = config.get('general.minimum_send_pow') + else: + minDifficulty = config.get('general.minimum_block_pow') + + retData = max(minDifficulty, math.floor(dataSize / 1000000)) + getDifficultyModifier() + return retData def getHashDifficulty(h): ''' @@ -55,6 +105,7 @@ class DataPOW: self.difficulty = 0 self.data = data self.threadCount = threadCount + self.rounds = 0 config.reload() if forceDifficulty == 0: @@ -96,6 +147,7 @@ class DataPOW: while self.hashing: rand = nacl.utils.random() token = nacl.hash.blake2b(rand + self.data).decode() + self.rounds += 1 #print(token) if self.puzzle == token[0:self.difficulty]: self.hashing = False @@ -106,6 +158,7 @@ class DataPOW: endTime = math.floor(time.time()) if self.reporting: logger.debug('Found token after %s seconds: %s' % (endTime - startTime, token), timestamp=True) + logger.debug('Round count: %s' % (self.rounds,)) self.result = (token, rand) def shutdown(self): @@ -146,18 +199,28 @@ class DataPOW: return result class POW: - def __init__(self, metadata, data, threadCount = 5): + def __init__(self, metadata, data, threadCount = 5, forceDifficulty=0, coreInst=None): self.foundHash = False self.difficulty = 0 self.data = data self.metadata = metadata self.threadCount = threadCount - dataLen = len(data) + len(json.dumps(metadata)) - self.difficulty = math.floor(dataLen / 1000000) - if self.difficulty <= 2: - self.difficulty = int(config.get('general.minimum_block_pow')) + try: + assert isinstance(coreInst, core.Core) + except AssertionError: + myCore = core.Core() + else: + myCore = coreInst + dataLen = len(data) + len(json.dumps(metadata)) + + if forceDifficulty > 0: + self.difficulty = forceDifficulty + else: + # Calculate difficulty. Dumb for now, may use good algorithm in the future. + self.difficulty = getDifficultyForNewBlock(dataLen) + try: self.data = self.data.encode() except AttributeError: @@ -167,8 +230,7 @@ class POW: self.mainHash = '0' * 64 self.puzzle = self.mainHash[0:min(self.difficulty, len(self.mainHash))] - - myCore = core.Core() + for i in range(max(1, threadCount)): t = threading.Thread(name = 'thread%s' % i, target = self.pow, args = (True,myCore)) t.start() diff --git a/onionr/onionrstorage.py b/onionr/onionrstorage.py new file mode 100644 index 00000000..fc460917 --- /dev/null +++ b/onionr/onionrstorage.py @@ -0,0 +1,31 @@ +''' + Onionr - P2P Anonymous Storage Network + + This file handles block storage, providing an abstraction for storing blocks between file system and database +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +import core +class OnionrStorage: + def __init__(self, coreInst): + assert isinstance(coreInst, core.Core) + self._core = coreInst + return + + def store(self, hash, data): + return + + def getData(self, hash): + return \ No newline at end of file diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 1037eb07..95c243f1 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -155,18 +155,21 @@ class OnionrUtils: ''' Send a command to the local http API server, securely. Intended for local clients, DO NOT USE for remote peers. ''' - config.reload() self.getTimeBypassToken() # TODO: URL encode parameters, just as an extra measure. May not be needed, but should be added regardless. hostname = '' + maxWait = 5 + waited = 0 while hostname == '': try: with open(self._core.privateApiHostFile, 'r') as host: hostname = host.read() except FileNotFoundError: - print('wat') time.sleep(1) + waited += 1 + if waited == maxWait: + return False if data != '': data = '&data=' + urllib.parse.quote_plus(data) payload = 'http://%s:%s/%s%s' % (hostname, config.get('client.client.port'), command, data) diff --git a/onionr/static-data/default_config.json b/onionr/static-data/default_config.json index 13436852..ed9270db 100644 --- a/onionr/static-data/default_config.json +++ b/onionr/static-data/default_config.json @@ -1,14 +1,13 @@ { "general" : { "dev_mode" : true, - "display_header" : true, - "minimum_block_pow": 5, - "minimum_send_pow": 5, + "display_header" : false, + "minimum_block_pow": 3, + "minimum_send_pow": 3, "socket_servers": false, "security_level": 0, "max_block_age": 2678400, - "public_key": "", - "use_new_api_server": false + "public_key": "" }, "www" : { @@ -70,7 +69,7 @@ }, "allocations" : { - "disk" : 10000000000, + "disk" : 100000000, "net_total" : 1000000000, "blockCache" : 5000000, "blockCacheTotal" : 50000000 diff --git a/onionr/static-data/www/board/board.js b/onionr/static-data/www/board/board.js new file mode 100644 index 00000000..7f513357 --- /dev/null +++ b/onionr/static-data/www/board/board.js @@ -0,0 +1,58 @@ +webpassword = '' +requested = [] + +document.getElementById('webpassWindow').style.display = 'block'; + +var windowHeight = window.innerHeight; +document.getElementById('webpassWindow').style.height = windowHeight + "px"; + +function httpGet(theUrl) { + var xmlHttp = new XMLHttpRequest() + xmlHttp.open( "GET", theUrl, false ) // false for synchronous request + xmlHttp.setRequestHeader('token', webpassword) + xmlHttp.send( null ) + if (xmlHttp.status == 200){ + return xmlHttp.responseText + } + else{ + return ""; + } +} +function appendMessages(msg){ + el = document.createElement('div') + el.className = 'entry' + el.innerText = msg + document.getElementById('feed').appendChild(el) + document.getElementById('feed').appendChild(document.createElement('br')) +} + +function getBlocks(){ + if (document.getElementById('none') !== null){ + document.getElementById('none').remove(); + + } + var feedText = httpGet('/getblocksbytype/txt') + var blockList = feedText.split(',') + for (i = 0; i < blockList.length; i++){ + if (! requested.includes(blockList[i])){ + bl = httpGet('/gethtmlsafeblockdata/' + blockList[i]) + appendMessages(bl) + requested.push(blockList[i]) + } + } +} + +document.getElementById('registerPassword').onclick = function(){ + webpassword = document.getElementById('webpassword').value + if (httpGet('/ping') === 'pong!'){ + document.getElementById('webpassWindow').style.display = 'none' + getBlocks() + } + else{ + alert('Sorry, but that password appears invalid.') + } +} + +document.getElementById('refreshFeed').onclick = function(){ + getBlocks() +} \ No newline at end of file diff --git a/onionr/static-data/www/board/index.html b/onionr/static-data/www/board/index.html new file mode 100644 index 00000000..df48e912 --- /dev/null +++ b/onionr/static-data/www/board/index.html @@ -0,0 +1,21 @@ + + + + + + OnionrBoard + + + + + + +
None Yet :)
+ + + \ No newline at end of file diff --git a/onionr/static-data/www/board/theme.css b/onionr/static-data/www/board/theme.css new file mode 100644 index 00000000..766e4407 --- /dev/null +++ b/onionr/static-data/www/board/theme.css @@ -0,0 +1,31 @@ +h1, h2, h3{ + font-family: sans-serif; +} +.hidden{ + display: none; +} +p{ + font-family: sans-serif; +} +#webpassWindow{ + background-color: black; + border: 1px solid black; + border-radius: 5px; + width: 100%; + z-index: 2; + color: white; + text-align: center; +} + +.entry{ + color: red; +} + +#feed{ + margin-left: 2%; + margin-right: 25%; + margin-top: 1em; + border: 2px solid black; + padding: 5px; + min-height: 50px; +} \ No newline at end of file diff --git a/onionr/storagecounter.py b/onionr/storagecounter.py index 4468dacc..863145f9 100644 --- a/onionr/storagecounter.py +++ b/onionr/storagecounter.py @@ -43,6 +43,11 @@ class StorageCounter: except FileNotFoundError: pass return retData + + def getPercent(self): + '''Return percent (decimal/float) of disk space we're using''' + amount = self.getAmount() + return round(amount / self._core.config.get('allocations.disk'), 2) def addBytes(self, amount): '''Record that we are now using more disk space, unless doing so would exceed configured max'''