kev
/
Onionr
1
0
Fork 0
Code Issues 24 Pull Requests Projects Releases 2 Wiki Activity

Merge branch 'dbstorage' of gitlab.com:beardog/Onionr into dbstorage

This commit is contained in:
Kevin Froman 2018-12-31 14:12:41 -06:00
parent fd4f1967e8 f53d3a11a6
commit a259ee4a09
17 changed files with 303 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.gitlab-ci.yml
View File

@ -1,6 +0,0 @@
test:
script:
- apt-get update -qy
- apt-get install -y python3-dev python3-pip tor
- pip3 install -r requirements.txt
- make test

8
.travis.yml
View File

@ -1,8 +0,0 @@
language: python
python:
- "3.6.4"
# install dependencies
install:
- sudo apt install tor
- pip install -r requirements.txt
script: make test

2
onionr-daemon-linux Normal file
View File

@ -0,0 +1,2 @@
#!/usr/bin/sh
nohup ./run-linux start & disown

60
onionr/api.py
View File

@ -17,7 +17,7 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import flask
import flask, cgi
from flask import request, Response, abort, send_from_directory
from gevent.pywsgi import WSGIServer
import sys, random, threading, hmac, hashlib, base64, time, math, os, json
@ -221,7 +221,7 @@ class API:
This initilization defines all of the API entry points and handlers for the endpoints and errors
This also saves the used host (random localhost IP address) to the data folder in host.txt
'''
# assert isinstance(onionrInst, onionr.Onionr)
# configure logger and stuff
onionr.Onionr.setupConfig('data/', self = self)
@ -234,6 +234,8 @@ class API:
bindPort = int(config.get('client.client.port', 59496))
self.bindPort = bindPort
self.whitelistEndpoints = ('site', 'www', 'onionrhome', 'board', 'boardContent', 'sharedContent')
self.clientToken = config.get('client.webpassword')
self.timeBypassToken = base64.b16encode(os.urandom(32)).decode()
@ -249,6 +251,8 @@ class API:
'''Validate request has set password and is the correct hostname'''
if request.host != '%s:%s' % (self.host, self.bindPort):
abort(403)
if request.endpoint in self.whitelistEndpoints:
return
try:
if not hmac.compare_digest(request.headers['token'], self.clientToken):
abort(403)
@ -257,7 +261,8 @@ class API:
@app.after_request
def afterReq(resp):
resp.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'"
#resp.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'"
resp.headers['Content-Security-Policy'] = "default-src 'none'; script-src 'self'; object-src 'none'; style-src 'self'; img-src 'self'; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'self'"
resp.headers['X-Frame-Options'] = 'deny'
resp.headers['X-Content-Type-Options'] = "nosniff"
resp.headers['X-API'] = onionr.API_VERSION
@ -265,20 +270,57 @@ class API:
resp.headers['Date'] = 'Thu, 1 Jan 1970 00:00:00 GMT' # Clock info is probably useful to attackers. Set to unix epoch.
return resp
@app.route('/board/', endpoint='board')
def loadBoard():
return send_from_directory('static-data/www/board/', "index.html")
@app.route('/board/<path:path>', endpoint='boardContent')
def boardContent(path):
return send_from_directory('static-data/www/board/', path)
@app.route('/shared/<path:path>', endpoint='sharedContent')
def sharedContent(path):
return send_from_directory('static-data/www/shared/', path)
@app.route('/www/<path:path>', endpoint='www')
def wwwPublic(path):
if not config.get("www.private.run", True):
abort(403)
return send_from_directory(config.get('www.private.path', 'static-data/www/private/'), path)
@app.route('/ping')
def ping():
return Response("pong!")
@app.route('/')
@app.route('/', endpoint='onionrhome')
def hello():
return Response("hello client")
return Response("Welcome to Onionr")
@app.route('/getblocksbytype/<name>')
def getBlocksByType(name):
blocks = self._core.getBlocksByType(name)
return Response(','.join(blocks))
@app.route('/gethtmlsafeblockdata/<name>')
def getData(name):
resp = ''
if self._core._utils.validateHash(name):
try:
resp = cgi.escape(Block(name).bcontent, quote=True)
except TypeError:
pass
else:
abort(404)
return Response(resp)
@app.route('/site/<name>')
def site():
bHash = block
@app.route('/site/<name>', endpoint='site')
def site(name):
bHash = name
resp = 'Not Found'
if self._core._utils.validateHash(bHash):
resp = Block(bHash).bcontent
try:
resp = Block(bHash).bcontent
except TypeError:
pass
try:
resp = base64.b64decode(resp)
except:

2
onionr/communicator2.py
View File

@ -102,7 +102,7 @@ class OnionrCommunicatorDaemon:
OnionrCommunicatorTimers(self, self.daemonTools.cooldownPeer, 30, requiresPeer=True)
OnionrCommunicatorTimers(self, self.uploadBlock, 10, requiresPeer=True, maxThreads=1)
OnionrCommunicatorTimers(self, self.daemonCommands, 6, maxThreads=1)
OnionrCommunicatorTimers(self, self.detectAPICrash, 5, maxThreads=1)
OnionrCommunicatorTimers(self, self.detectAPICrash, 30, maxThreads=1)
deniableBlockTimer = OnionrCommunicatorTimers(self, self.daemonTools.insertDeniableBlock, 180, requiresPeer=True, maxThreads=1)
netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600)

24
onionr/core.py
View File

@ -681,7 +681,10 @@ class Core:
Inserts a block into the network
encryptType must be specified to encrypt a block
'''
allocationReachedMessage = 'Cannot insert block, disk allocation reached.'
if self._utils.storageCounter.isFull():
logger.error(allocationReachedMessage)
return False
retData = False
# check nonce
dataNonce = self._utils.bytesToStr(self._crypto.sha3Hash(data))
@ -775,13 +778,18 @@ class Core:
proof = onionrproofs.POW(metadata, data)
payload = proof.waitForResult()
if payload != False:
retData = self.setData(payload)
# Tell the api server through localCommand to wait for the daemon to upload this block to make stastical analysis more difficult
self._utils.localCommand('waitforshare/' + retData)
self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
#self.setBlockType(retData, meta['type'])
self._utils.processBlockMetadata(retData)
self.daemonQueueAdd('uploadBlock', retData)
try:
retData = self.setData(payload)
except onionrexceptions.DiskAllocationReached:
logger.error(allocationReachedMessage)
retData = False
else:
# Tell the api server through localCommand to wait for the daemon to upload this block to make stastical analysis more difficult
self._utils.localCommand('waitforshare/' + retData)
self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
#self.setBlockType(retData, meta['type'])
self._utils.processBlockMetadata(retData)
self.daemonQueueAdd('uploadBlock', retData)
if retData != False:
events.event('insertBlock', onionr = None, threaded = False)

43
onionr/onionr.py
View File

@ -351,46 +351,9 @@ class Onionr:
except IndexError:
logger.error('Friend ID is required.')
except onionrexceptions.KeyNotKnown:
logger.error('That peer is not in our database')
else:
if action == 'add':
friend.setTrust(1)
logger.info('Added %s as friend.' % (friend.publicKey,))
else:
friend.setTrust(0)
logger.info('Removed %s as friend.' % (friend.publicKey,))
else:
logger.info('Syntax: friend add/remove/list [address]')
def friendCmd(self):
'''List, add, or remove friend(s)
Changes their peer DB entry.
'''
friend = ''
try:
# Get the friend command
action = sys.argv[2]
except IndexError:
logger.info('Syntax: friend add/remove/list [address]')
else:
action = action.lower()
if action == 'list':
# List out peers marked as our friend
for friend in self.onionrCore.listPeers(randomOrder=False, trust=1):
if friend == self.onionrCore._crypto.pubKey: # do not list our key
continue
friendProfile = onionrusers.OnionrUser(self.onionrCore, friend)
logger.info(friend + ' - ' + friendProfile.getName())
elif action in ('add', 'remove'):
try:
friend = sys.argv[3]
if not self.onionrUtils.validatePubKey(friend):
raise onionrexceptions.InvalidPubkey('Public key is invalid')
self.onionrCore.addPeer(friend)
friend = onionrusers.OnionrUser(self.onionrCore, friend)
except IndexError:
logger.error('Friend ID is required.')
else:
finally:
if action == 'add':
friend.setTrust(1)
logger.info('Added %s as friend.' % (friend.publicKey,))
@ -400,7 +363,6 @@ class Onionr:
else:
logger.info('Syntax: friend add/remove/list [address]')
def deleteRunFiles(self):
try:
os.remove(self.onionrCore.publicApiHostFile)
@ -719,7 +681,6 @@ class Onionr:
'''
Starts the Onionr communication daemon
'''
communicatorDaemon = './communicator2.py'
# remove runcheck if it exists

4
onionr/onionrblockapi.py
View File

@ -245,8 +245,8 @@ class Block:
blockFile.write(self.getRaw().encode())
else:
self.hash = self.getCore().insertBlock(self.getContent(), header = self.getType(), sign = sign, meta = self.getMetadata(), expire = self.getExpire())
self.update()
if self.hash != False:
self.update()
return self.getHash()
else:

5
onionr/onionrcrypto.py
View File

@ -268,8 +268,9 @@ class OnionrCrypto:
blockHash = blockHash.decode() # bytes on some versions for some reason
except AttributeError:
pass
difficulty = math.floor(dataLen / 1000000)
difficulty = onionrproofs.getDifficultyForNewBlock(blockContent, ourBlock=False)
if difficulty < int(config.get('general.minimum_block_pow')):
difficulty = int(config.get('general.minimum_block_pow'))
mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode()

78
onionr/onionrproofs.py
View File

@ -19,7 +19,57 @@
'''
import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, logger, sys, base64, json
import core, config
import core, onionrutils, config
import onionrblockapi
def getDifficultyModifier(coreOrUtilsInst=None):
'''Accepts a core or utils instance returns
the difficulty modifier for block storage based
on a variety of factors, currently only disk use.
'''
classInst = coreOrUtilsInst
retData = 0
if isinstance(classInst, core.Core):
useFunc = classInst._utils.storageCounter.getPercent
elif isinstance(classInst, onionrutils.OnionrUtils):
useFunc = classInst.storageCounter.getPercent
else:
useFunc = core.Core()._utils.storageCounter.getPercent
percentUse = useFunc()
if percentUse >= 0.50:
retData += 1
elif percentUse >= 0.75:
retData += 2
elif percentUse >= 0.95:
retData += 3
return retData
def getDifficultyForNewBlock(data, ourBlock=True):
'''
Get difficulty for block. Accepts size in integer, Block instance, or str/bytes full block contents
'''
retData = 0
dataSize = 0
if isinstance(data, onionrblockapi.Block):
dataSize = len(data.getRaw().encode('utf-8'))
elif isinstance(data, str):
dataSize = len(data.encode('utf-8'))
elif isinstance(data, bytes):
dataSize = len(data)
elif isinstance(data, int):
dataSize = data
else:
raise ValueError('not Block, str, or int')
if ourBlock:
minDifficulty = config.get('general.minimum_send_pow')
else:
minDifficulty = config.get('general.minimum_block_pow')
retData = max(minDifficulty, math.floor(dataSize / 1000000)) + getDifficultyModifier()
return retData
def getHashDifficulty(h):
'''
@ -55,6 +105,7 @@ class DataPOW:
self.difficulty = 0
self.data = data
self.threadCount = threadCount
self.rounds = 0
config.reload()
if forceDifficulty == 0:
@ -96,6 +147,7 @@ class DataPOW:
while self.hashing:
rand = nacl.utils.random()
token = nacl.hash.blake2b(rand + self.data).decode()
self.rounds += 1
#print(token)
if self.puzzle == token[0:self.difficulty]:
self.hashing = False
@ -106,6 +158,7 @@ class DataPOW:
endTime = math.floor(time.time())
if self.reporting:
logger.debug('Found token after %s seconds: %s' % (endTime - startTime, token), timestamp=True)
logger.debug('Round count: %s' % (self.rounds,))
self.result = (token, rand)
def shutdown(self):
@ -146,18 +199,28 @@ class DataPOW:
return result
class POW:
def __init__(self, metadata, data, threadCount = 5):
def __init__(self, metadata, data, threadCount = 5, forceDifficulty=0, coreInst=None):
self.foundHash = False
self.difficulty = 0
self.data = data
self.metadata = metadata
self.threadCount = threadCount
dataLen = len(data) + len(json.dumps(metadata))
self.difficulty = math.floor(dataLen / 1000000)
if self.difficulty <= 2:
self.difficulty = int(config.get('general.minimum_block_pow'))
try:
assert isinstance(coreInst, core.Core)
except AssertionError:
myCore = core.Core()
else:
myCore = coreInst
dataLen = len(data) + len(json.dumps(metadata))
if forceDifficulty > 0:
self.difficulty = forceDifficulty
else:
# Calculate difficulty. Dumb for now, may use good algorithm in the future.
self.difficulty = getDifficultyForNewBlock(dataLen)
try:
self.data = self.data.encode()
except AttributeError:
@ -167,8 +230,7 @@ class POW:
self.mainHash = '0' * 64
self.puzzle = self.mainHash[0:min(self.difficulty, len(self.mainHash))]
myCore = core.Core()
for i in range(max(1, threadCount)):
t = threading.Thread(name = 'thread%s' % i, target = self.pow, args = (True,myCore))
t.start()

31
onionr/onionrstorage.py Normal file
View File

@ -0,0 +1,31 @@
'''
Onionr - P2P Anonymous Storage Network
This file handles block storage, providing an abstraction for storing blocks between file system and database
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import core
class OnionrStorage:
def __init__(self, coreInst):
assert isinstance(coreInst, core.Core)
self._core = coreInst
return
def store(self, hash, data):
return
def getData(self, hash):
return

7
onionr/onionrutils.py
View File

@ -155,18 +155,21 @@ class OnionrUtils:
'''
Send a command to the local http API server, securely. Intended for local clients, DO NOT USE for remote peers.
'''
config.reload()
self.getTimeBypassToken()
# TODO: URL encode parameters, just as an extra measure. May not be needed, but should be added regardless.
hostname = ''
maxWait = 5
waited = 0
while hostname == '':
try:
with open(self._core.privateApiHostFile, 'r') as host:
hostname = host.read()
except FileNotFoundError:
print('wat')
time.sleep(1)
waited += 1
if waited == maxWait:
return False
if data != '':
data = '&data=' + urllib.parse.quote_plus(data)
payload = 'http://%s:%s/%s%s' % (hostname, config.get('client.client.port'), command, data)

11
onionr/static-data/default_config.json
View File

@ -1,14 +1,13 @@
{
"general" : {
"dev_mode" : true,
"display_header" : true,
"minimum_block_pow": 5,
"minimum_send_pow": 5,
"display_header" : false,
"minimum_block_pow": 3,
"minimum_send_pow": 3,
"socket_servers": false,
"security_level": 0,
"max_block_age": 2678400,
"public_key": "",
"use_new_api_server": false
"public_key": ""
},
"www" : {
@ -70,7 +69,7 @@
},
"allocations" : {
"disk" : 10000000000,
"disk" : 100000000,
"net_total" : 1000000000,
"blockCache" : 5000000,
"blockCacheTotal" : 50000000

58
onionr/static-data/www/board/board.js Normal file
View File

@ -0,0 +1,58 @@
webpassword = ''
requested = []
document.getElementById('webpassWindow').style.display = 'block';
var windowHeight = window.innerHeight;
document.getElementById('webpassWindow').style.height = windowHeight + "px";
function httpGet(theUrl) {
var xmlHttp = new XMLHttpRequest()
xmlHttp.open( "GET", theUrl, false ) // false for synchronous request
xmlHttp.setRequestHeader('token', webpassword)
xmlHttp.send( null )
if (xmlHttp.status == 200){
return xmlHttp.responseText
}
else{
return "";
}
}
function appendMessages(msg){
el = document.createElement('div')
el.className = 'entry'
el.innerText = msg
document.getElementById('feed').appendChild(el)
document.getElementById('feed').appendChild(document.createElement('br'))
}
function getBlocks(){
if (document.getElementById('none') !== null){
document.getElementById('none').remove();
}
var feedText = httpGet('/getblocksbytype/txt')
var blockList = feedText.split(',')
for (i = 0; i < blockList.length; i++){
if (! requested.includes(blockList[i])){
bl = httpGet('/gethtmlsafeblockdata/' + blockList[i])
appendMessages(bl)
requested.push(blockList[i])
}
}
}
document.getElementById('registerPassword').onclick = function(){
webpassword = document.getElementById('webpassword').value
if (httpGet('/ping') === 'pong!'){
document.getElementById('webpassWindow').style.display = 'none'
getBlocks()
}
else{
alert('Sorry, but that password appears invalid.')
}
}
document.getElementById('refreshFeed').onclick = function(){
getBlocks()
}

21
onionr/static-data/www/board/index.html Normal file
View File

@ -0,0 +1,21 @@
<!DOCTYPE HTML>
<html>
<head>
<meta charset='utf-8'>
<title>
OnionrBoard
</title>
<link rel='stylesheet' href='theme.css'>
</head>
<body>
<div id='webpassWindow' class='hidden'>
<p>Welcome to OnionrBoard</p>
<p>Please enter the webpassword. You can get this from running the 'details' command in Onionr.</p>
<input id='webpassword' type='password' placeholder="Web password for daemon" value='7AF13568657CE63D6DB7E686BF05537D36598ED739B21E3F023E3FD3DEA2FC8F'>
<button id='registerPassword'>Unlock Onionr</button>
</div>
<input type='button' id='refreshFeed' value='Refresh Feed'>
<div id='feed'><span id='none'>None Yet :)</span></div>
<script src='board.js'></script>
</body>
</html>

31
onionr/static-data/www/board/theme.css Normal file
View File

@ -0,0 +1,31 @@
h1, h2, h3{
font-family: sans-serif;
}
.hidden{
display: none;
}
p{
font-family: sans-serif;
}
#webpassWindow{
background-color: black;
border: 1px solid black;
border-radius: 5px;
width: 100%;
z-index: 2;
color: white;
text-align: center;
}
.entry{
color: red;
}
#feed{
margin-left: 2%;
margin-right: 25%;
margin-top: 1em;
border: 2px solid black;
padding: 5px;
min-height: 50px;
}

5
onionr/storagecounter.py
View File

@ -43,6 +43,11 @@ class StorageCounter:
except FileNotFoundError:
pass
return retData
def getPercent(self):
'''Return percent (decimal/float) of disk space we're using'''
amount = self.getAmount()
return round(amount / self._core.config.get('allocations.disk'), 2)
def addBytes(self, amount):
'''Record that we are now using more disk space, unless doing so would exceed configured max'''