Browse Source

Merge Tempblocks

tags/0.0.0
Kevin 1 year ago
parent
commit
44d545684a
40 changed files with 1636 additions and 446 deletions
  1. +2
    -0
      .gitignore
  2. +0
    -3
      .gitmodules
  3. +1
    -1
      Dockerfile
  4. +1
    -0
      RUN-LINUX.sh
  5. +14
    -11
      onionr/api.py
  6. +65
    -35
      onionr/communicator2.py
  7. +8
    -1
      onionr/config.py
  8. +132
    -123
      onionr/core.py
  9. +42
    -4
      onionr/dbcreator.py
  10. +50
    -14
      onionr/netcontroller.py
  11. +129
    -55
      onionr/onionr.py
  12. +8
    -7
      onionr/onionrblacklist.py
  13. +28
    -8
      onionr/onionrblockapi.py
  14. +49
    -0
      onionr/onionrchat.py
  15. +20
    -51
      onionr/onionrcrypto.py
  16. +64
    -6
      onionr/onionrdaemontools.py
  17. +15
    -1
      onionr/onionrexceptions.py
  18. +0
    -19
      onionr/onionri2p.py
  19. +4
    -2
      onionr/onionrpeers.py
  20. +9
    -2
      onionr/onionrplugins.py
  21. +28
    -6
      onionr/onionrproofs.py
  22. +171
    -0
      onionr/onionrsockets.py
  23. +189
    -0
      onionr/onionrusers.py
  24. +88
    -39
      onionr/onionrutils.py
  25. +1
    -1
      onionr/onionrvalues.py
  26. +0
    -2
      onionr/static-data/bootstrap-nodes.txt
  27. +5
    -0
      onionr/static-data/default-plugins/cliui/info.json
  28. +133
    -0
      onionr/static-data/default-plugins/cliui/main.py
  29. +5
    -0
      onionr/static-data/default-plugins/encrypt/info.json
  30. +117
    -0
      onionr/static-data/default-plugins/encrypt/main.py
  31. +2
    -2
      onionr/static-data/default-plugins/flow/main.py
  32. +5
    -0
      onionr/static-data/default-plugins/metadataprocessor/info.json
  33. +103
    -0
      onionr/static-data/default-plugins/metadataprocessor/main.py
  34. +69
    -15
      onionr/static-data/default-plugins/pms/main.py
  35. +62
    -0
      onionr/static-data/default-plugins/pms/sentboxdb.py
  36. +4
    -2
      onionr/static-data/default_config.json
  37. +2
    -2
      onionr/static-data/index.html
  38. +1
    -31
      onionr/tests.py
  39. +3
    -3
      requirements.txt
  40. +7
    -0
      setprofile.sh

+ 2
- 0
.gitignore View File

@@ -13,3 +13,5 @@ onionr/data-encrypted.dat
onionr/.onionr-lock
core
.vscode/*
venv/*
onionr/fs*

+ 0
- 3
.gitmodules View File

@@ -1,3 +0,0 @@
[submodule "onionr/bitpeer"]
path = onionr/bitpeer
url = https://github.com/beardog108/bitpeer.py

+ 1
- 1
Dockerfile View File

@@ -4,7 +4,7 @@ FROM ubuntu:bionic
ENV HOME /root

#Install needed packages
RUN apt update && apt install -y python3 python3-dev python3-pip tor locales nano
RUN apt update && apt install -y python3 python3-dev python3-pip tor locales nano sqlite3

RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
locale-gen


+ 1
- 0
RUN-LINUX.sh View File

@@ -1,3 +1,4 @@
#!/bin/sh
cd "$(dirname "$0")"
cd onionr/
./onionr.py "$@"

+ 14
- 11
onionr/api.py View File

@@ -1,5 +1,5 @@
'''
Onionr - P2P Microblogging Platform & Social network
Onionr - P2P Anonymous Storage Network

This file handles all incoming http requests to the client, using Flask
'''
@@ -20,9 +20,9 @@
import flask
from flask import request, Response, abort, send_from_directory
from multiprocessing import Process
from gevent.wsgi import WSGIServer
from gevent.pywsgi import WSGIServer
import sys, random, threading, hmac, hashlib, base64, time, math, os, json
from core import Core
import core
from onionrblockapi import Block
import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config

@@ -37,6 +37,9 @@ class API:
'''
Validate that the client token matches the given token
'''
if len(self.clientToken) == 0:
logger.error("client password needs to be set")
return False
try:
if not hmac.compare_digest(self.clientToken, token):
return False
@@ -69,7 +72,7 @@ class API:
logger.debug('%s not in %s' % (path, mimetypes))
return 'text/plain'

def __init__(self, debug):
def __init__(self, debug, API_VERSION):
'''
Initialize the api server, preping variables for later use

@@ -88,7 +91,7 @@ class API:

self.debug = debug
self._privateDelayTime = 3
self._core = Core()
self._core = core.Core()
self._crypto = onionrcrypto.OnionrCrypto(self._core)
self._utils = onionrutils.OnionrUtils(self._core)
app = flask.Flask(__name__)
@@ -102,7 +105,7 @@ class API:
self.mimeType = 'text/plain'
self.overrideCSP = False

with open('data/time-bypass.txt', 'w') as bypass:
with open(self._core.dataDir + 'time-bypass.txt', 'w') as bypass:
bypass.write(self.timeBypassToken)

if not debug and not self._developmentMode:
@@ -111,7 +114,7 @@ class API:
else:
self.host = '127.0.0.1'

with open('data/host.txt', 'w') as file:
with open(self._core.dataDir + 'host.txt', 'w') as file:
file.write(self.host)

@app.before_request
@@ -133,14 +136,14 @@ class API:
resp.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'"
resp.headers['X-Frame-Options'] = 'deny'
resp.headers['X-Content-Type-Options'] = "nosniff"
resp.headers['server'] = 'Onionr'
resp.headers['api'] = API_VERSION

# reset to text/plain to help prevent browser attacks
self.mimeType = 'text/plain'
self.overrideCSP = False

return resp
@app.route('/www/private/<path:path>')
def www_private(path):
startTime = math.floor(time.time())
@@ -466,7 +469,7 @@ class API:
elif action == 'getData':
resp = ''
if self._utils.validateHash(data):
if os.path.exists('data/blocks/' + data + '.dat'):
if os.path.exists(self._core.dataDir + 'blocks/' + data + '.dat'):
block = Block(hash=data.encode(), core=self._core)
resp = base64.b64encode(block.getRaw().encode()).decode()
if len(resp) == 0:
@@ -515,7 +518,7 @@ class API:
while len(self._core.hsAddress) == 0:
self._core.refreshFirstStartVars()
time.sleep(0.5)
self.http_server = WSGIServer((self.host, bindPort), app)
self.http_server = WSGIServer((self.host, bindPort), app, log=None)
self.http_server.serve_forever()
except KeyboardInterrupt:
pass


+ 65
- 35
onionr/communicator2.py View File

@@ -1,6 +1,6 @@
#!/usr/bin/env python3
'''
Onionr - P2P Microblogging Platform & Social network.
Onionr - P2P Anonymous Storage Network

This file contains both the OnionrCommunicate class for communcating with peers
and code to operate as a daemon, getting commands from the command queue database (see core.Core.daemonQueue)
@@ -19,9 +19,10 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import sys, os, core, config, json, requests, time, logger, threading, base64, onionr
import sys, os, core, config, json, requests, time, logger, threading, base64, onionr, uuid
import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block
import onionrdaemontools
import onionrdaemontools, onionrsockets, onionrchat
from dependencies import secrets
from defusedxml import minidom

class OnionrCommunicatorDaemon:
@@ -51,6 +52,8 @@ class OnionrCommunicatorDaemon:
# lists of connected peers and peers we know we can't reach currently
self.onlinePeers = []
self.offlinePeers = []
self.cooldownPeer = {}
self.connectTimes = {}
self.peerProfiles = [] # list of peer's profiles (onionrpeers.PeerProfile instances)

# amount of threads running by name, used to prevent too many
@@ -77,29 +80,39 @@ class OnionrCommunicatorDaemon:
#self.daemonTools = onionrdaemontools.DaemonTools(self)
self.daemonTools = onionrdaemontools.DaemonTools(self)

self._chat = onionrchat.OnionrChat(self)

if debug or developmentMode:
OnionrCommunicatorTimers(self, self.heartbeat, 10)
OnionrCommunicatorTimers(self, self.heartbeat, 30)

# Set timers, function reference, seconds
# requiresPeer True means the timer function won't fire if we have no connected peers
# TODO: make some of these timer counts configurable
OnionrCommunicatorTimers(self, self.daemonCommands, 5)
OnionrCommunicatorTimers(self, self.detectAPICrash, 5)
peerPoolTimer = OnionrCommunicatorTimers(self, self.getOnlinePeers, 60)
peerPoolTimer = OnionrCommunicatorTimers(self, self.getOnlinePeers, 60, maxThreads=1)
OnionrCommunicatorTimers(self, self.lookupBlocks, self._core.config.get('timers.lookupBlocks'), requiresPeer=True, maxThreads=1)
OnionrCommunicatorTimers(self, self.getBlocks, self._core.config.get('timers.getBlocks'), requiresPeer=True)
OnionrCommunicatorTimers(self, self.clearOfflinePeer, 58)
OnionrCommunicatorTimers(self, self.daemonTools.cleanOldBlocks, 65)
OnionrCommunicatorTimers(self, self.lookupKeys, 60, requiresPeer=True)
OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True)
OnionrCommunicatorTimers(self, self.daemonTools.cooldownPeer, 30, requiresPeer=True)
netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600)
announceTimer = OnionrCommunicatorTimers(self, self.daemonTools.announceNode, 305, requiresPeer=True, maxThreads=1)
cleanupTimer = OnionrCommunicatorTimers(self, self.peerCleanup, 300, requiresPeer=True)
forwardSecrecyTimer = OnionrCommunicatorTimers(self, self.daemonTools.cleanKeys, 15)

# set loop to execute instantly to load up peer pool (replaced old pool init wait)
peerPoolTimer.count = (peerPoolTimer.frequency - 1)
cleanupTimer.count = (cleanupTimer.frequency - 60)
announceTimer.count = (cleanupTimer.frequency - 60)
#forwardSecrecyTimer.count = (forwardSecrecyTimer.frequency - 990)

self.socketServer = threading.Thread(target=onionrsockets.OnionrSocketServer, args=(self._core,))
self.socketServer.start()
self.socketClient = onionrsockets.OnionrSocketClient(self._core)

# Loads chat messages into memory
threading.Thread(target=self._chat.chatHandler).start()

# Main daemon loop, mainly for calling timers, don't do any complex operations here to avoid locking
try:
@@ -114,21 +127,10 @@ class OnionrCommunicatorDaemon:
pass

logger.info('Goodbye.')
self._core.killSockets = True
self._core._utils.localCommand('shutdown') # shutdown the api
time.sleep(0.5)

def lookupKeys(self):
'''Lookup new keys'''
logger.debug('Looking up new keys...')
tryAmount = 1
for i in range(tryAmount): # amount of times to ask peers for new keys
# Download new key list from random online peers
peer = self.pickOnlinePeer()
newKeys = self.peerAction(peer, action='kex')
self._core._utils.mergeKeys(newKeys)
self.decrementThreadCount('lookupKeys')
return

def lookupAdders(self):
'''Lookup new peer addresses'''
logger.info('LOOKING UP NEW ADDRESSES')
@@ -147,10 +149,13 @@ class OnionrCommunicatorDaemon:
newBlocks = ''
existingBlocks = self._core.getBlockList()
triedPeers = [] # list of peers we've tried this time around
maxBacklog = 1560 # Max amount of *new* block hashes to have already in queue, to avoid memory exhaustion
for i in range(tryAmount):
# check if disk allocation is used
if len(self.blockQueue) >= maxBacklog:
break
if not self.isOnline:
break
# check if disk allocation is used
if self._core._utils.storageCounter.isFull():
logger.debug('Not looking up new blocks due to maximum amount of allowed disk space used')
break
@@ -180,6 +185,7 @@ class OnionrCommunicatorDaemon:
if not i in existingBlocks:
# if block does not exist on disk and is not already in block queue
if i not in self.blockQueue and not self._core._blacklist.inBlacklist(i):
# TODO ensure block starts with minimum difficulty before adding to queue
self.blockQueue.append(i) # add blocks to download queue
self.decrementThreadCount('lookupBlocks')
return
@@ -207,7 +213,7 @@ class OnionrCommunicatorDaemon:
logger.info("Attempting to download %s..." % blockHash)
peerUsed = self.pickOnlinePeer()
content = self.peerAction(peerUsed, 'getData', data=blockHash) # block content from random peer (includes metadata)
if content != False:
if content != False and len(content) > 0:
try:
content = content.encode()
except AttributeError:
@@ -253,7 +259,10 @@ class OnionrCommunicatorDaemon:
onionrpeers.PeerProfiles(peerUsed, self._core).addScore(-50)
logger.warn('Block hash validation failed for ' + blockHash + ' got ' + tempHash)
if removeFromQueue:
self.blockQueue.remove(blockHash) # remove from block queue both if success or false
try:
self.blockQueue.remove(blockHash) # remove from block queue both if success or false
except ValueError:
pass
self.currentDownloading.remove(blockHash)
self.decrementThreadCount('getBlocks')
return
@@ -296,7 +305,7 @@ class OnionrCommunicatorDaemon:
'''Manages the self.onlinePeers attribute list, connects to more peers if we have none connected'''

logger.info('Refreshing peer pool.')
maxPeers = 6
maxPeers = int(config.get('peers.maxConnect'))
needed = maxPeers - len(self.onlinePeers)

for i in range(needed):
@@ -339,7 +348,7 @@ class OnionrCommunicatorDaemon:
for address in peerList:
if not config.get('tor.v3onions') and len(address) == 62:
continue
if len(address) == 0 or address in tried or address in self.onlinePeers:
if len(address) == 0 or address in tried or address in self.onlinePeers or address in self.cooldownPeer:
continue
if self.shutdown:
return
@@ -348,6 +357,7 @@ class OnionrCommunicatorDaemon:
time.sleep(0.1)
if address not in self.onlinePeers:
self.onlinePeers.append(address)
self.connectTimes[address] = self._core._utils.getEpoch()
retData = address
# add peer to profile list if they're not in it
@@ -362,6 +372,17 @@ class OnionrCommunicatorDaemon:
logger.debug('Failed to connect to ' + address)
return retData

def removeOnlinePeer(self, peer):
'''Remove an online peer'''
try:
del self.connectTimes[peer]
except KeyError:
pass
try:
self.onlinePeers.remove(peer)
except ValueError:
pass

def peerCleanup(self):
'''This just calls onionrpeers.cleanupPeers, which removes dead or bad peers (offline too long, too slow)'''
onionrpeers.peerCleanup(self._core)
@@ -393,8 +414,9 @@ class OnionrCommunicatorDaemon:
if retData == False:
try:
self.getPeerProfileInstance(peer).addScore(-10)
self.onlinePeers.remove(peer)
self.getOnlinePeers() # Will only add a new peer to pool if needed
self.removeOnlinePeer(peer)
if action != 'ping':
self.getOnlinePeers() # Will only add a new peer to pool if needed
except ValueError:
pass
else:
@@ -430,16 +452,15 @@ class OnionrCommunicatorDaemon:
if cmd[0] == 'shutdown':
self.shutdown = True
elif cmd[0] == 'announceNode':
self.announce(cmd[1])
if len(self.onlinePeers) > 0:
self.announce(cmd[1])
else:
logger.warn("Not introducing, since I have no connected nodes.")
elif cmd[0] == 'runCheck':
logger.debug('Status check; looks good.')
open('data/.runcheck', 'w+').close()
open(self._core.dataDir + '.runcheck', 'w+').close()
elif cmd[0] == 'connectedPeers':
self.printOnlinePeers()
elif cmd[0] == 'kex':
for i in self.timers:
if i.timerFunction.__name__ == 'lookupKeys':
i.count = (i.frequency - 1)
elif cmd[0] == 'pex':
for i in self.timers:
if i.timerFunction.__name__ == 'lookupAdders':
@@ -447,6 +468,17 @@ class OnionrCommunicatorDaemon:
elif cmd[0] == 'uploadBlock':
self.blockToUpload = cmd[1]
threading.Thread(target=self.uploadBlock).start()
elif cmd[0] == 'startSocket':
# Create our own socket server
socketInfo = json.loads(cmd[1])
socketInfo['id'] = uuid.uuid4()
self._core.startSocket = socketInfo
elif cmd[0] == 'addSocket':
# Socket server was created for us
socketInfo = json.loads(cmd[1])
peer = socketInfo['peer']
reason = socketInfo['reason']
threading.Thread(target=self.socketClient.startSocket, args=(peer, reason)).start()
else:
logger.info('Recieved daemonQueue command:' + cmd[0])

@@ -476,9 +508,7 @@ class OnionrCommunicatorDaemon:

def announce(self, peer):
'''Announce to peers our address'''
if self.daemonTools.announceNode():
logger.info('Successfully introduced node to ' + peer)
else:
if self.daemonTools.announceNode() == False:
logger.warn('Could not introduce node.')

def detectAPICrash(self):


+ 8
- 1
onionr/config.py View File

@@ -20,7 +20,14 @@

import os, json, logger

_configfile = os.path.abspath('data/config.json')
try:
dataDir = os.environ['ONIONR_HOME']
if not dataDir.endswith('/'):
dataDir += '/'
except KeyError:
dataDir = 'data/'
_configfile = os.path.abspath(dataDir + 'config.json')
_config = {}

def get(key, default = None):


+ 132
- 123
onionr/core.py View File

@@ -1,5 +1,5 @@
'''
Onionr - P2P Microblogging Platform & Social network
Onionr - P2P Anonymous Storage Network

Core Onionr library, useful for external programs. Handles peer & data processing
'''
@@ -17,11 +17,11 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import sqlite3, os, sys, time, math, base64, tarfile, getpass, simplecrypt, hashlib, nacl, logger, json, netcontroller, math, config
import sqlite3, os, sys, time, math, base64, tarfile, nacl, logger, json, netcontroller, math, config
from onionrblockapi import Block

import onionrutils, onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions, onionrvalues
import onionrblacklist
import onionrblacklist, onionrchat, onionrusers
import dbcreator
if sys.version_info < (3, 6):
try:
@@ -35,34 +35,52 @@ class Core:
'''
Initialize Core Onionr library
'''

try:
self.dataDir = os.environ['ONIONR_HOME']
if not self.dataDir.endswith('/'):
self.dataDir += '/'
except KeyError:
self.dataDir = 'data/'
try:
self.queueDB = 'data/queue.db'
self.peerDB = 'data/peers.db'
self.blockDB = 'data/blocks.db'
self.blockDataLocation = 'data/blocks/'
self.addressDB = 'data/address.db'
self.queueDB = self.dataDir + 'queue.db'
self.peerDB = self.dataDir + 'peers.db'
self.blockDB = self.dataDir + 'blocks.db'
self.blockDataLocation = self.dataDir + 'blocks/'
self.addressDB = self.dataDir + 'address.db'
self.hsAddress = ''
self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt'
self.bootstrapList = []
self.requirements = onionrvalues.OnionrValues()
self.torPort = torPort
self.dataNonceFile = 'data/block-nonces.dat'
self.dataNonceFile = self.dataDir + 'block-nonces.dat'
self.dbCreate = dbcreator.DBCreator(self)
self.forwardKeysFile = self.dataDir + 'forward-keys.db'

# Socket data, defined here because of multithreading constraints with gevent
self.killSockets = False
self.startSocket = {}
self.socketServerConnData = {}
self.socketReasons = {}
self.socketServerResponseData = {}

self.usageFile = 'data/disk-usage.txt'
self.usageFile = self.dataDir + 'disk-usage.txt'
self.config = config

self.maxBlockSize = 10000000 # max block size in bytes

if not os.path.exists('data/'):
os.mkdir('data/')
if not os.path.exists('data/blocks/'):
os.mkdir('data/blocks/')
if not os.path.exists(self.dataDir):
os.mkdir(self.dataDir)
if not os.path.exists(self.dataDir + 'blocks/'):
os.mkdir(self.dataDir + 'blocks/')
if not os.path.exists(self.blockDB):
self.createBlockDB()
if not os.path.exists(self.forwardKeysFile):
self.dbCreate.createForwardKeyDB()

if os.path.exists('data/hs/hostname'):
with open('data/hs/hostname', 'r') as hs:
if os.path.exists(self.dataDir + '/hs/hostname'):
with open(self.dataDir + '/hs/hostname', 'r') as hs:
self.hsAddress = hs.read().strip()

# Load bootstrap address list
@@ -87,8 +105,8 @@ class Core:

def refreshFirstStartVars(self):
'''Hack to refresh some vars which may not be set on first start'''
if os.path.exists('data/hs/hostname'):
with open('data/hs/hostname', 'r') as hs:
if os.path.exists(self.dataDir + '/hs/hostname'):
with open(self.dataDir + '/hs/hostname', 'r') as hs:
self.hsAddress = hs.read().strip()

def addPeer(self, peerID, powID, name=''):
@@ -102,10 +120,12 @@ class Core:
logger.warn("POW token for pubkey base64 representation exceeded 120 bytes, is " + str(sys.getsizeof(powID)))
return False

conn = sqlite3.connect(self.peerDB)
events.event('pubkey_add', data = {'key': peerID}, onionr = None)
conn = sqlite3.connect(self.peerDB, timeout=10)
hashID = self._crypto.pubKeyHashID(peerID)
c = conn.cursor()
t = (peerID, name, 'unknown', hashID, powID)
t = (peerID, name, 'unknown', hashID, powID, 0)

for i in c.execute("SELECT * FROM PEERS where id = '" + peerID + "';"):
try:
@@ -116,7 +136,7 @@ class Core:
pass
except IndexError:
pass
c.execute('INSERT INTO peers (id, name, dateSeen, pow, hashID) VALUES(?, ?, ?, ?, ?);', t)
c.execute('INSERT INTO peers (id, name, dateSeen, pow, hashID, trust) VALUES(?, ?, ?, ?, ?, ?);', t)
conn.commit()
conn.close()

@@ -126,11 +146,11 @@ class Core:
'''
Add an address to the address database (only tor currently)
'''
if address == config.get('i2p.ownAddr', None):
if address == config.get('i2p.ownAddr', None) or address == self.hsAddress:

return False
if self._utils.validateID(address):
conn = sqlite3.connect(self.addressDB)
conn = sqlite3.connect(self.addressDB, timeout=10)
c = conn.cursor()
# check if address is in database
# this is safe to do because the address is validated above, but we strip some chars here too just in case
@@ -162,7 +182,7 @@ class Core:
Remove an address from the address database
'''
if self._utils.validateID(address):
conn = sqlite3.connect(self.addressDB)
conn = sqlite3.connect(self.addressDB, timeout=10)
c = conn.cursor()
t = (address,)
c.execute('Delete from adders where address=?;', t)
@@ -181,13 +201,13 @@ class Core:
**You may want blacklist.addToDB(blockHash)
'''
if self._utils.validateHash(block):
conn = sqlite3.connect(self.blockDB)
conn = sqlite3.connect(self.blockDB, timeout=10)
c = conn.cursor()
t = (block,)
c.execute('Delete from hashes where hash=?;', t)
conn.commit()
conn.close()
blockFile = 'data/blocks/' + block + '.dat'
blockFile = self.dataDir + '/blocks/' + block + '.dat'
dataSize = 0
try:
''' Get size of data when loaded as an object/var, rather than on disk,
@@ -228,7 +248,7 @@ class Core:
raise Exception('Block db does not exist')
if self._utils.hasBlock(newHash):
return
conn = sqlite3.connect(self.blockDB)
conn = sqlite3.connect(self.blockDB, timeout=10)
c = conn.cursor()
currentTime = self._utils.getEpoch()
if selfInsert or dataSaved:
@@ -256,14 +276,6 @@ class Core:

return data

def _getSha3Hash(self, data):
hasher = hashlib.sha3_256()
if not type(data) is bytes:
data = data.encode()
hasher.update(data)
dataHash = hasher.hexdigest()
return dataHash

def setData(self, data):
'''
Set the data assciated with a hash
@@ -274,7 +286,7 @@ class Core:
if not type(data) is bytes:
data = data.encode()
dataHash = self._getSha3Hash(data)
dataHash = self._crypto.sha3Hash(data)

if type(dataHash) is bytes:
dataHash = dataHash.decode()
@@ -287,7 +299,7 @@ class Core:
blockFile = open(blockFileName, 'wb')
blockFile.write(data)
blockFile.close()
conn = sqlite3.connect(self.blockDB)
conn = sqlite3.connect(self.blockDB, timeout=10)
c = conn.cursor()
c.execute("UPDATE hashes SET dataSaved=1 WHERE hash = '" + dataHash + "';")
conn.commit()
@@ -299,42 +311,6 @@ class Core:

return dataHash

def dataDirEncrypt(self, password):
'''
Encrypt the data directory on Onionr shutdown
'''
if os.path.exists('data.tar'):
os.remove('data.tar')
tar = tarfile.open("data.tar", "w")
for name in ['data']:
tar.add(name)
tar.close()
tarData = open('data.tar', 'r', encoding = "ISO-8859-1").read()
encrypted = simplecrypt.encrypt(password, tarData)
open('data-encrypted.dat', 'wb').write(encrypted)
os.remove('data.tar')

return

def dataDirDecrypt(self, password):
'''
Decrypt the data directory on startup
'''
if not os.path.exists('data-encrypted.dat'):
return (False, 'encrypted archive does not exist')
data = open('data-encrypted.dat', 'rb').read()
try:
decrypted = simplecrypt.decrypt(password, data)
except simplecrypt.DecryptionException:
return (False, 'wrong password (or corrupted archive)')
else:
open('data.tar', 'wb').write(decrypted)
tar = tarfile.open('data.tar')
tar.extractall()
tar.close()

return (True, '')

def daemonQueue(self):
'''
Gives commands to the communication proccess/daemon by reading an sqlite3 database
@@ -343,16 +319,16 @@ class Core:
'''
retData = False
if not os.path.exists(self.queueDB):
self.makeDaemonDB()
self.dbCreate.createDaemonDB()
else:
conn = sqlite3.connect(self.queueDB)
conn = sqlite3.connect(self.queueDB, timeout=10)
c = conn.cursor()
try:
for row in c.execute('SELECT command, data, date, min(ID) FROM commands group by id'):
retData = row
break
except sqlite3.OperationalError:
self.makeDaemonDB()
self.dbCreate.createDaemonDB()
else:
if retData != False:
c.execute('DELETE FROM commands WHERE id=?;', (retData[3],))
@@ -363,38 +339,32 @@ class Core:

return retData

def makeDaemonDB(self):
'''generate the daemon queue db'''
conn = sqlite3.connect(self.queueDB)
c = conn.cursor()
# Create table
c.execute('''CREATE TABLE commands
(id integer primary key autoincrement, command text, data text, date text)''')
conn.commit()
conn.close()

def daemonQueueAdd(self, command, data=''):
'''
Add a command to the daemon queue, used by the communication daemon (communicator.py)
'''
retData = True
# Intended to be used by the web server
date = self._utils.getEpoch()
conn = sqlite3.connect(self.queueDB)
conn = sqlite3.connect(self.queueDB, timeout=10)
c = conn.cursor()
t = (command, data, date)
c.execute('INSERT INTO commands (command, data, date) VALUES(?, ?, ?)', t)
conn.commit()
conn.close()

try:
c.execute('INSERT INTO commands (command, data, date) VALUES(?, ?, ?)', t)
conn.commit()
conn.close()
except sqlite3.OperationalError:
retData = False
self.daemonQueue()
events.event('queue_push', data = {'command': command, 'data': data}, onionr = None)

return
return retData

def clearDaemonQueue(self):
'''
Clear the daemon queue (somewhat dangerous)
'''
conn = sqlite3.connect(self.queueDB)
conn = sqlite3.connect(self.queueDB, timeout=10)
c = conn.cursor()
try:
c.execute('DELETE FROM commands;')
@@ -410,7 +380,7 @@ class Core:
'''
Return a list of addresses
'''
conn = sqlite3.connect(self.addressDB)
conn = sqlite3.connect(self.addressDB, timeout=10)
c = conn.cursor()
if randomOrder:
addresses = c.execute('SELECT * FROM adders ORDER BY RANDOM();')
@@ -422,19 +392,23 @@ class Core:
conn.close()
return addressList

def listPeers(self, randomOrder=True, getPow=False):
def listPeers(self, randomOrder=True, getPow=False, trust=0):
'''
Return a list of public keys (misleading function name)

randomOrder determines if the list should be in a random order
trust sets the minimum trust to list
'''
conn = sqlite3.connect(self.peerDB)
conn = sqlite3.connect(self.peerDB, timeout=10)
c = conn.cursor()
payload = ""
if trust not in (0, 1, 2):
logger.error('Tried to select invalid trust.')
return
if randomOrder:
payload = 'SELECT * FROM peers ORDER BY RANDOM();'
payload = 'SELECT * FROM peers where trust >= %s ORDER BY RANDOM();' % (trust,)
else:
payload = 'SELECT * FROM peers;'
payload = 'SELECT * FROM peers where trust >= %s;' % (trust,)
peerList = []
for i in c.execute(payload):
try:
@@ -462,18 +436,17 @@ class Core:
id text 0
name text, 1
adders text, 2
forwardKey text, 3
dateSeen not null, 4
bytesStored int, 5
trust int 6
pubkeyExchanged int 7
hashID text 8
pow text 9
'''
conn = sqlite3.connect(self.peerDB)
dateSeen not null, 3
bytesStored int, 4
trust int 5
pubkeyExchanged int 6
hashID text 7
pow text 8
'''
conn = sqlite3.connect(self.peerDB, timeout=10)
c = conn.cursor()
command = (peer,)
infoNumbers = {'id': 0, 'name': 1, 'adders': 2, 'forwardKey': 3, 'dateSeen': 4, 'bytesStored': 5, 'trust': 6, 'pubkeyExchanged': 7, 'hashID': 8}
infoNumbers = {'id': 0, 'name': 1, 'adders': 2, 'dateSeen': 3, 'bytesStored': 4, 'trust': 5, 'pubkeyExchanged': 6, 'hashID': 7}
info = infoNumbers[info]
iterCount = 0
retVal = ''
@@ -492,7 +465,7 @@ class Core:
'''
Update a peer for a key
'''
conn = sqlite3.connect(self.peerDB)
conn = sqlite3.connect(self.peerDB, timeout=10)
c = conn.cursor()
command = (data, peer)
# TODO: validate key on whitelist
@@ -516,7 +489,7 @@ class Core:
failure int 6
lastConnect 7
'''
conn = sqlite3.connect(self.addressDB)
conn = sqlite3.connect(self.addressDB, timeout=10)
c = conn.cursor()
command = (address,)
infoNumbers = {'address': 0, 'type': 1, 'knownPeer': 2, 'speed': 3, 'success': 4, 'DBHash': 5, 'failure': 6, 'lastConnect': 7}
@@ -537,7 +510,7 @@ class Core:
'''
Update an address for a key
'''
conn = sqlite3.connect(self.addressDB)
conn = sqlite3.connect(self.addressDB, timeout=10)
c = conn.cursor()
command = (data, address)
# TODO: validate key on whitelist
@@ -553,7 +526,7 @@ class Core:
'''
Get list of our blocks
'''
conn = sqlite3.connect(self.blockDB)
conn = sqlite3.connect(self.blockDB, timeout=10)
c = conn.cursor()
if unsaved:
execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();'
@@ -570,7 +543,7 @@ class Core:
'''
Returns the date a block was received
'''
conn = sqlite3.connect(self.blockDB)
conn = sqlite3.connect(self.blockDB, timeout=10)
c = conn.cursor()
execute = 'SELECT dateReceived FROM hashes WHERE hash=?;'
args = (blockHash,)
@@ -584,7 +557,7 @@ class Core:
'''
Returns a list of blocks by the type
'''
conn = sqlite3.connect(self.blockDB)
conn = sqlite3.connect(self.blockDB, timeout=10)
c = conn.cursor()
if orderDate:
execute = 'SELECT hash FROM hashes WHERE dataType=? ORDER BY dateReceived;'
@@ -595,15 +568,28 @@ class Core:
for row in c.execute(execute, args):
for i in row:
rows.append(i)

return rows

def getExpiredBlocks(self):
'''Returns a list of expired blocks'''
conn = sqlite3.connect(self.blockDB, timeout=10)
c = conn.cursor()
date = int(self._utils.getEpoch())

execute = 'SELECT hash FROM hashes WHERE expire <= %s ORDER BY dateReceived;' % (date,)

rows = list()
for row in c.execute(execute):
for i in row:
rows.append(i)
return rows

def setBlockType(self, hash, blockType):
'''
Sets the type of block
'''

conn = sqlite3.connect(self.blockDB)
conn = sqlite3.connect(self.blockDB, timeout=10)
c = conn.cursor()
c.execute("UPDATE hashes SET dataType='" + blockType + "' WHERE hash = '" + hash + "';")
conn.commit()
@@ -623,12 +609,13 @@ class Core:
sig - optional signature by the author (not optional if author is specified)
author - multi-round partial sha3-256 hash of authors public key
dateClaimed - timestamp claimed inside the block, only as trustworthy as the block author is
expire - expire date for a block
'''

if key not in ('dateReceived', 'decrypted', 'dataType', 'dataFound', 'dataSaved', 'sig', 'author', 'dateClaimed'):
if key not in ('dateReceived', 'decrypted', 'dataType', 'dataFound', 'dataSaved', 'sig', 'author', 'dateClaimed', 'expire'):
return False

conn = sqlite3.connect(self.blockDB)
conn = sqlite3.connect(self.blockDB, timeout=10)
c = conn.cursor()
args = (data, hash)
c.execute("UPDATE hashes SET " + key + " = ? where hash = ?;", args)
@@ -636,7 +623,7 @@ class Core:
conn.close()
return True

def insertBlock(self, data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = None):
def insertBlock(self, data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = None, expire=None):
'''
Inserts a block into the network
encryptType must be specified to encrypt a block
@@ -673,8 +660,6 @@ class Core:
meta['type'] = header
meta['type'] = str(meta['type'])

jsonMeta = json.dumps(meta)

if encryptType in ('asym', 'sym', ''):
metadata['encryptType'] = encryptType
else:
@@ -684,7 +669,20 @@ class Core:
data = data.encode()
except AttributeError:
pass
# sign before encrypt, as unauthenticated crypto should not be a problem here

if encryptType == 'asym':
try:
forwardEncrypted = onionrusers.OnionrUser(self, asymPeer).forwardEncrypt(data)
data = forwardEncrypted[0]
meta['forwardEnc'] = True
except onionrexceptions.InvalidPubkey:
onionrusers.OnionrUser(self, asymPeer).generateForwardKey()
else:
logger.info(forwardEncrypted)
onionrusers.OnionrUser(self, asymPeer).generateForwardKey()
fsKey = onionrusers.OnionrUser(self, asymPeer).getGeneratedForwardKeys()[0]
meta['newFSKey'] = fsKey[0]
jsonMeta = json.dumps(meta)
if sign:
signature = self._crypto.edSign(jsonMeta.encode() + data, key=self._crypto.privKey, encodeResult=True)
signer = self._crypto.pubKey
@@ -692,8 +690,11 @@ class Core:
if len(jsonMeta) > 1000:
raise onionrexceptions.InvalidMetadata('meta in json encoded form must not exceed 1000 bytes')

user = onionrusers.OnionrUser(self, symKey)

# encrypt block metadata/sig/content
if encryptType == 'sym':

if len(symKey) < self.requirements.passwordLength:
raise onionrexceptions.SecurityError('Weak encryption key')
jsonMeta = self._crypto.symmetricEncrypt(jsonMeta, key=symKey, returnEncoded=True).decode()
@@ -702,6 +703,8 @@ class Core:
signer = self._crypto.symmetricEncrypt(signer, key=symKey, returnEncoded=True).decode()
elif encryptType == 'asym':
if self._utils.validatePubKey(asymPeer):
# Encrypt block data with forward secrecy key first, but not meta
jsonMeta = json.dumps(meta)
jsonMeta = self._crypto.pubKeyEncrypt(jsonMeta, asymPeer, encodedData=True, anonymous=True).decode()
data = self._crypto.pubKeyEncrypt(data, asymPeer, encodedData=True, anonymous=True).decode()
signature = self._crypto.pubKeyEncrypt(signature, asymPeer, encodedData=True, anonymous=True).decode()
@@ -714,6 +717,11 @@ class Core:
metadata['sig'] = signature
metadata['signer'] = signer
metadata['time'] = str(self._utils.getEpoch())

# ensure expire is integer and of sane length
if type(expire) is not type(None):
assert len(str(int(expire))) < 14
metadata['expire'] = expire
# send block data (and metadata) to POW module to get tokenized block data
proof = onionrproofs.POW(metadata, data)
@@ -721,7 +729,8 @@ class Core:
if payload != False:
retData = self.setData(payload)
self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
self.setBlockType(retData, meta['type'])
#self.setBlockType(retData, meta['type'])
self._utils.processBlockMetadata(retData)
self.daemonQueueAdd('uploadBlock', retData)

if retData != False:


+ 42
- 4
onionr/dbcreator.py View File

@@ -61,8 +61,6 @@ class DBCreator:
ID text not null,
name text,
adders text,
blockDBHash text,
forwardKey text,
dateSeen not null,
bytesStored int,
trust int,
@@ -70,6 +68,12 @@ class DBCreator:
hashID text,
pow text not null);
''')
c.execute('''CREATE TABLE forwardKeys(
peerKey text not null,
forwardKey text not null,
date int not null,
expire int not null
);''')
conn.commit()
conn.close()
return
@@ -87,6 +91,7 @@ class DBCreator:
sig - optional signature by the author (not optional if author is specified)
author - multi-round partial sha3-256 hash of authors public key
dateClaimed - timestamp claimed inside the block, only as trustworthy as the block author is
expire int - block expire date in epoch
'''
if os.path.exists(self.core.blockDB):
raise Exception("Block database already exists")
@@ -101,9 +106,42 @@ class DBCreator:
dataSaved int,
sig text,
author text,
dateClaimed int
dateClaimed int,
expire int
);
''')
conn.commit()
conn.close()
return
return

def createForwardKeyDB(self):
'''
Create the forward secrecy key db (*for *OUR* keys*)
'''
if os.path.exists(self.core.forwardKeysFile):
raise Exception("Block database already exists")
conn = sqlite3.connect(self.core.forwardKeysFile)
c = conn.cursor()
c.execute('''CREATE TABLE myForwardKeys(
peer text not null,
publickey text not null,
privatekey text not null,
date int not null,
expire int not null
);
''')
conn.commit()
conn.close()
return
def createDaemonDB(self):
'''
Create the daemon queue database
'''
conn = sqlite3.connect(self.core.queueDB, timeout=10)
c = conn.cursor()
# Create table
c.execute('''CREATE TABLE commands
(id integer primary key autoincrement, command text, data text, date text)''')
conn.commit()
conn.close()

+ 50
- 14
onionr/netcontroller.py View File

@@ -18,21 +18,37 @@
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''

import subprocess, os, random, sys, logger, time, signal, config
import subprocess, os, random, sys, logger, time, signal, config, base64
from stem.control import Controller
from onionrblockapi import Block
from dependencies import secrets
class NetController:
'''
This class handles hidden service setup on Tor and I2P
'''

def __init__(self, hsPort):
self.torConfigLocation = 'data/torrc'
try:
self.dataDir = os.environ['ONIONR_HOME']
if not self.dataDir.endswith('/'):
self.dataDir += '/'
except KeyError:
self.dataDir = 'data/'

self.torConfigLocation = self.dataDir + 'torrc'
self.readyState = False
self.socksPort = random.randint(1024, 65535)
self.hsPort = hsPort
self._torInstnace = ''
self.myID = ''

if os.path.exists('./tor'):
self.torBinary = './tor'
elif os.path.exists('/usr/bin/tor'):
self.torBinary = '/usr/bin/tor'
else:
self.torBinary = 'tor'

config.reload()
'''
if os.path.exists(self.torConfigLocation):
@@ -52,13 +68,33 @@ class NetController:
if config.get('tor.v3onions'):
hsVer = 'HiddenServiceVersion 3'
logger.info('Using v3 onions :)')

if os.path.exists(self.torConfigLocation):
os.remove(self.torConfigLocation)

# Set the Tor control password. Meant to make it harder to manipulate our Tor instance
plaintext = base64.b64encode(os.urandom(50)).decode()
config.set('tor.controlpassword', plaintext, savefile=True)
config.set('tor.socksport', self.socksPort, savefile=True)

controlPort = random.randint(1025, 65535)

config.set('tor.controlPort', controlPort, savefile=True)

hashedPassword = subprocess.Popen([self.torBinary, '--hash-password', plaintext], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
for line in iter(hashedPassword.stdout.readline, b''):
password = line.decode()
if 'warn' not in password:
break

torrcData = '''SocksPort ''' + str(self.socksPort) + '''
HiddenServiceDir data/hs/
HiddenServiceDir ''' + self.dataDir + '''hs/
\n''' + hsVer + '''\n
HiddenServicePort 80 127.0.0.1:''' + str(self.hsPort) + '''
DataDirectory data/tordata/
DataDirectory ''' + self.dataDir + '''tordata/
CookieAuthentication 1
ControlPort ''' + str(controlPort) + '''
HashedControlPassword ''' + str(password) + '''
'''
torrc = open(self.torConfigLocation, 'w')
torrc.write(torrcData)
@@ -74,20 +110,20 @@ DataDirectory data/tordata/
self.generateTorrc()

if os.path.exists('./tor'):
torBinary = './tor'
self.torBinary = './tor'
elif os.path.exists('/usr/bin/tor'):
torBinary = '/usr/bin/tor'
self.torBinary = '/usr/bin/tor'
else:
torBinary = 'tor'
self.torBinary = 'tor'

try:
tor = subprocess.Popen([torBinary, '-f', self.torConfigLocation], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
tor = subprocess.Popen([self.torBinary, '-f', self.torConfigLocation], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
except FileNotFoundError:
logger.fatal("Tor was not found in your path or the Onionr directory. Please install Tor and try again.")
sys.exit(1)
else:
# Test Tor Version
torVersion = subprocess.Popen([torBinary, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
torVersion = subprocess.Popen([self.torBinary, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
for line in iter(torVersion.stdout.readline, b''):
if 'Tor 0.2.' in line.decode():
logger.warn("Running 0.2.x Tor series, no support for v3 onion peers")
@@ -111,11 +147,11 @@ DataDirectory data/tordata/
logger.debug('Finished starting Tor.', timestamp=True)
self.readyState = True

myID = open('data/hs/hostname', 'r')
myID = open(self.dataDir + 'hs/hostname', 'r')
self.myID = myID.read().replace('\n', '')
myID.close()

torPidFile = open('data/torPid.txt', 'w')
torPidFile = open(self.dataDir + 'torPid.txt', 'w')
torPidFile.write(str(tor.pid))
torPidFile.close()

@@ -127,7 +163,7 @@ DataDirectory data/tordata/
'''

try:
pid = open('data/torPid.txt', 'r')
pid = open(self.dataDir + 'torPid.txt', 'r')
pidN = pid.read()
pid.close()
except FileNotFoundError:
@@ -140,7 +176,7 @@ DataDirectory data/tordata/

try:
os.kill(int(pidN), signal.SIGTERM)
os.remove('data/torPid.txt')
os.remove(self.dataDir + 'torPid.txt')
except ProcessLookupError:
pass
except FileNotFoundError:


+ 129
- 55
onionr/onionr.py View File

@@ -1,6 +1,6 @@
#!/usr/bin/env python3
'''
Onionr - P2P Microblogging Platform & Social network.
Onionr - P2P Anonymous Storage Network

Onionr is the name for both the protocol and the original/reference software.

@@ -26,13 +26,13 @@ if sys.version_info[0] == 2 or sys.version_info[1] < 5:
print('Error, Onionr requires Python 3.4+')
sys.exit(1)
import os, base64, random, getpass, shutil, subprocess, requests, time, platform, datetime, re, json, getpass, sqlite3
import webbrowser
from threading import Thread
import api, core, config, logger, onionrplugins as plugins, onionrevents as events
import onionrutils
from onionrutils import OnionrUtils
from netcontroller import NetController
from onionrblockapi import Block
import onionrproofs
import onionrproofs, onionrexceptions, onionrusers

try:
from urllib3.contrib.socks import SOCKSProxyManager
@@ -40,9 +40,9 @@ except ImportError:
raise Exception("You need the PySocks module (for use with socks5 proxy to use Tor)")

ONIONR_TAGLINE = 'Anonymous P2P Platform - GPLv3 - https://Onionr.VoidNet.Tech'
ONIONR_VERSION = '0.2.0' # for debugging and stuff
ONIONR_VERSION = '0.3.0' # for debugging and stuff
ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION)
API_VERSION = '4' # increments of 1; only change when something fundemental about how the API works changes. This way other nodes know how to communicate without learning too much information about you.
API_VERSION = '5' # increments of 1; only change when something fundemental about how the API works changes. This way other nodes know how to communicate without learning too much information about you.

class Onionr:
def __init__(self):
@@ -50,23 +50,31 @@ class Onionr:
Main Onionr class. This is for the CLI program, and does not handle much of the logic.
In general, external programs and plugins should not use this class.
'''
self.userRunDir = os.getcwd() # Directory user runs the program from
try:
os.chdir(sys.path[0])
except FileNotFoundError:
pass

try:
self.dataDir = os.environ['ONIONR_HOME']
if not self.dataDir.endswith('/'):
self.dataDir += '/'
except KeyError:
self.dataDir = 'data/'

# Load global configuration data

data_exists = os.path.exists('data/')
data_exists = os.path.exists(self.dataDir)

if not data_exists:
os.mkdir('data/')
os.mkdir(self.dataDir)

if os.path.exists('static-data/default_config.json'):
config.set_config(json.loads(open('static-data/default_config.json').read())) # this is the default config, it will be overwritten if a config file already exists. Else, it saves it
else:
# the default config file doesn't exist, try hardcoded config
config.set_config({'dev_mode': True, 'log': {'file': {'output': True, 'path': 'data/output.log'}, 'console': {'output': True, 'color': True}}})
config.set_config({'dev_mode': True, 'log': {'file': {'output': True, 'path': self.dataDir + 'output.log'}, 'console': {'output': True, 'color': True}}})
if not data_exists:
config.save()
config.reload() # this will read the configuration file into memory
@@ -78,7 +86,7 @@ class Onionr:
settings = settings | logger.OUTPUT_TO_CONSOLE
if config.get('log.file.output', True):
settings = settings | logger.OUTPUT_TO_FILE
logger.set_file(config.get('log.file.path', '/tmp/onionr.log'))
logger.set_file(config.get('log.file.path', '/tmp/onionr.log').replace('data/', self.dataDir))
logger.set_settings(settings)

if str(config.get('general.dev_mode', True)).lower() == 'true':
@@ -89,37 +97,27 @@ class Onionr:
logger.set_level(logger.LEVEL_INFO)

self.onionrCore = core.Core()
self.onionrUtils = OnionrUtils(self.onionrCore)
self.onionrUtils = onionrutils.OnionrUtils(self.onionrCore)

# Handle commands

self.debug = False # Whole application debugging

if os.path.exists('data-encrypted.dat'):
while True:
print('Enter password to decrypt:')
password = getpass.getpass()
result = self.onionrCore.dataDirDecrypt(password)
if os.path.exists('data/'):
break
else:
logger.error('Failed to decrypt: ' + result[1], timestamp = False)
else:
# If data folder does not exist
if not data_exists:
if not os.path.exists('data/blocks/'):
os.mkdir('data/blocks/')

# Copy default plugins into plugins folder
if not os.path.exists(plugins.get_plugins_folder()):
if os.path.exists('static-data/default-plugins/'):
names = [f for f in os.listdir("static-data/default-plugins/") if not os.path.isfile(f)]
shutil.copytree('static-data/default-plugins/', plugins.get_plugins_folder())

# Enable plugins
for name in names:
if not name in plugins.get_enabled_plugins():
plugins.enable(name, self)
# If data folder does not exist
if not data_exists:
if not os.path.exists(self.dataDir + 'blocks/'):
os.mkdir(self.dataDir + 'blocks/')

# Copy default plugins into plugins folder
if not os.path.exists(plugins.get_plugins_folder()):
if os.path.exists('static-data/default-plugins/'):
names = [f for f in os.listdir("static-data/default-plugins/") if not os.path.isfile(f)]
shutil.copytree('static-data/default-plugins/', plugins.get_plugins_folder())

# Enable plugins
for name in names:
if not name in plugins.get_enabled_plugins():
plugins.enable(name, self)

for name in plugins.get_enabled_plugins():
if not os.path.exists(plugins.get_plugin_data_folder(name)):
@@ -190,6 +188,10 @@ class Onionr:

'add-file': self.addFile,
'addfile': self.addFile,

'get-file': self.getFile,
'getfile': self.getFile,

'listconn': self.listConn,

'import-blocks': self.onionrUtils.importNewBlocks,
@@ -210,7 +212,11 @@ class Onionr:
'getpass': self.printWebPassword,
'get-pass': self.printWebPassword,
'getpasswd': self.printWebPassword,
'get-passwd': self.printWebPassword
'get-passwd': self.printWebPassword,

'chat': self.startChat,

'friend': self.friendCmd
}

self.cmdhelp = {
@@ -228,12 +234,14 @@ class Onionr:
'add-peer': 'Adds a peer to database',
'list-peers': 'Displays a list of peers',
'add-file': 'Create an Onionr block from a file',
'get-file': 'Get a file from Onionr blocks',
'import-blocks': 'import blocks from the disk (Onionr is transport-agnostic!)',
'listconn': 'list connected peers',
'kex': 'exchange keys with peers (done automatically)',
'pex': 'exchange addresses with peers (done automatically)',
'blacklist-block': 'deletes a block by hash and permanently removes it from your node',
'introduce': 'Introduce your node to the public Onionr network',
'friend': '[add|remove] [public key/id]'
}

# initialize plugins
@@ -247,19 +255,64 @@ class Onionr:
finally:
self.execute(command)

if not self._developmentMode:
encryptionPassword = self.onionrUtils.getPassword('Enter password to encrypt directory: ')
self.onionrCore.dataDirEncrypt(encryptionPassword)
shutil.rmtree('data/')

return

'''
THIS SECTION HANDLES THE COMMANDS
'''

def startChat(self):
try:
data = json.dumps({'peer': sys.argv[2], 'reason': 'chat'})
except IndexError:
logger.error('Must specify peer to chat with.')
else:
self.onionrCore.daemonQueueAdd('startSocket', data)

def getCommands(self):
return self.cmds
def friendCmd(self):
'''List, add, or remove friend(s)
Changes their peer DB entry.
'''
friend = ''
try:
# Get the friend command
action = sys.argv[2]
except IndexError:
logger.info('Syntax: friend add/remove/list [address]')
else:
action = action.lower()
if action == 'list':
# List out peers marked as our friend
for friend in self.onionrCore.listPeers(randomOrder=False, trust=1):
if friend == self.onionrCore._crypto.pubKey: # do not list our key
continue
friendProfile = onionrusers.OnionrUser(self.onionrCore, friend)
logger.info(friend + ' - ' + friendProfile.getName())
elif action in ('add', 'remove'):
try:
friend = sys.argv[3]
if not self.onionrUtils.validatePubKey(friend):
raise onionrexceptions.InvalidPubkey('Public key is invalid')
if friend not in self.onionrCore.listPeers():
raise onionrexceptions.KeyNotKnown
friend = onionrusers.OnionrUser(self.onionrCore, friend)
except IndexError:
logger.error('Friend ID is required.')
except onionrexceptions.KeyNotKnown:
logger.error('That peer is not in our database')
else:
if action == 'add':
friend.setTrust(1)
logger.info('Added %s as friend.' % (friend.publicKey,))
else:
friend.setTrust(0)
logger.info('Removed %s as friend.' % (friend.publicKey,))
else:
logger.info('Syntax: friend add/remove/list [address]')


def banBlock(self):
try:
@@ -567,7 +620,7 @@ class Onionr:
'''
communicatorDaemon = './communicator2.py'

apiThread = Thread(target=api.API, args=(self.debug,))
apiThread = Thread(target=api.API, args=(self.debug,API_VERSION))
apiThread.start()
try:
time.sleep(3)
@@ -587,7 +640,7 @@ class Onionr:
logger.info('Our Public key: ' + self.onionrCore._crypto.pubKey)
time.sleep(1)
#TODO make runable on windows
subprocess.Popen([communicatorDaemon, "run", str(net.socksPort)])
communicatorProc = subprocess.Popen([communicatorDaemon, "run", str(net.socksPort)])
# Print nice header thing :)
if config.get('general.display_header', True):
self.header()
@@ -596,6 +649,9 @@ class Onionr:
try:
while True:
time.sleep(5)
# Break if communicator process ends, so we don't have left over processes
if communicatorProc.poll() is not None:
break
except KeyboardInterrupt:
self.onionrCore.daemonQueueAdd('shutdown')
self.onionrUtils.localCommand('shutdown')
@@ -630,26 +686,23 @@ class Onionr:
# define stats messages here
totalBlocks = len(Block.getBlocks())
signedBlocks = len(Block.getBlocks(signed = True))
powToken = self.onionrCore._crypto.pubKeyPowToken
messages = {
# info about local client
'Onionr Daemon Status' : ((logger.colors.fg.green + 'Online') if self.onionrUtils.isCommunicatorRunning(timeout = 2) else logger.colors.fg.red + 'Offline'),
'Onionr Daemon Status' : ((logger.colors.fg.green + 'Online') if self.onionrUtils.isCommunicatorRunning(timeout = 9) else logger.colors.fg.red + 'Offline'),
'Public Key' : self.onionrCore._crypto.pubKey,
'POW Token' : powToken,
'Combined' : self.onionrCore._crypto.pubKey + '-' + powToken,
'Human readable public key' : self.onionrCore._utils.getHumanReadableID(),
'Node Address' : self.get_hostname(),

# file and folder size stats
'div1' : True, # this creates a solid line across the screen, a div
'Total Block Size' : onionrutils.humanSize(onionrutils.size('data/blocks/')),
'Total Plugin Size' : onionrutils.humanSize(onionrutils.size('data/plugins/')),
'Log File Size' : onionrutils.humanSize(onionrutils.size('data/output.log')),
'Total Block Size' : onionrutils.humanSize(onionrutils.size(self.dataDir + 'blocks/')),
'Total Plugin Size' : onionrutils.humanSize(onionrutils.size(self.dataDir + 'plugins/')),
'Log File Size' : onionrutils.humanSize(onionrutils.size(self.dataDir + 'output.log')),

# count stats
'div2' : True,
'Known Peers Count' : str(len(self.onionrCore.listPeers()) - 1),
'Enabled Plugins Count' : str(len(config.get('plugins.enabled', list()))) + ' / ' + str(len(os.listdir('data/plugins/'))),
'Enabled Plugins Count' : str(len(config.get('plugins.enabled', list()))) + ' / ' + str(len(os.listdir(self.dataDir + 'plugins/'))),
'Known Blocks Count' : str(totalBlocks),
'Percent Blocks Signed' : str(round(100 * signedBlocks / max(totalBlocks, 1), 2)) + '%'
}
@@ -715,7 +768,7 @@ class Onionr:

def get_hostname(self):
try:
with open('./data/hs/hostname', 'r') as hostname:
with open('./' + self.dataDir + 'hs/hostname', 'r') as hostname:
return hostname.read().strip()
except Exception:
return None
@@ -735,6 +788,27 @@ class Onionr:

return columns

def getFile(self):
'''
Get a file from onionr blocks
'''
try:
fileName = sys.argv[2]
bHash = sys.argv[3]
except IndexError:
logger.error("Syntax %s %s" % (sys.argv[0], '/path/to/filename <blockhash>'))
else:
print(fileName)
contents = None
if os.path.exists(fileName):
logger.error("File already exists")
return
if not self.onionrUtils.validateHash(bHash):
logger.error('Block hash is invalid')
return
Block.mergeChain(bHash, fileName)
return

def addFile(self):
'''
Adds a file to the onionr network
@@ -745,8 +819,9 @@ class Onionr:
contents = None

if not os.path.exists(filename):
logger.warn('That file does not exist. Improper path?')

logger.error('That file does not exist. Improper path (specify full path)?')
return
logger.info('Adding file... this might take a long time.')
try:
blockhash = Block.createChain(file = filename)
logger.info('File %s saved in block %s.' % (filename, blockhash))
@@ -756,7 +831,6 @@ class Onionr:
logger.error('%s add-file <filename>' % sys.argv[0], timestamp = False)

def openUI(self):
import webbrowser
url = 'http://127.0.0.1:%s/ui/index.html?timingToken=%s' % (config.get('client.port', 59496), self.onionrUtils.getTimeBypassToken())

print('Opening %s ...' % url)


+ 8
- 7
onionr/onionrblacklist.py View File

@@ -1,5 +1,5 @@
'''
Onionr - P2P Microblogging Platform & Social network.
Onionr - P2P Anonymous Storage Network

This file handles maintenence of a blacklist database, for blocks and peers
'''
@@ -20,7 +20,7 @@
import sqlite3, os, logger
class OnionrBlackList:
def __init__(self, coreInst):
self.blacklistDB = 'data/blacklist.db'
self.blacklistDB = coreInst.dataDir + 'blacklist.db'
self._core = coreInst
if not os.path.exists(self.blacklistDB):
@@ -32,7 +32,8 @@ class OnionrBlackList:
retData = False
if not hashed.isalnum():
raise Exception("Hashed data is not alpha numeric")

if len(hashed) > 64:
raise Exception("Hashed data is too large")
for i in self._dbExecute("select * from blacklist where hash='%s'" % (hashed,)):
retData = True # this only executes if an entry is present by that hash
break
@@ -95,9 +96,8 @@ class OnionrBlackList:
'''
# we hash the data so we can remove data entirely from our node's disk
hashed = self._core._utils.bytesToStr(self._core._crypto.sha3Hash(data))

if self.inBlacklist(hashed):
return
if len(hashed) > 64:
raise Exception("Hashed data is too large")

if not hashed.isalnum():
raise Exception("Hashed data is not alpha numeric")
@@ -109,7 +109,8 @@ class OnionrBlackList:
int(expire)
except ValueError:
raise Exception("expire is not int")
#TODO check for length sanity
if self.inBlacklist(hashed):
return
insert = (hashed,)
blacklistDate = self._core._utils.getEpoch()
self._dbExecute("insert into blacklist (hash, dataType, blacklistDate, expire) VALUES('%s', %s, %s, %s);" % (hashed, dataType, blacklistDate, expire))

+ 28
- 8
onionr/onionrblockapi.py View File

@@ -1,5 +1,5 @@
'''
Onionr - P2P Microblogging Platform & Social network.
Onionr - P2P Anonymous Storage Network

This class contains the OnionrBlocks class which is a class for working with Onionr blocks
'''
@@ -18,14 +18,14 @@
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''

import core as onionrcore, logger, config, onionrexceptions, nacl.exceptions
import core as onionrcore, logger, config, onionrexceptions, nacl.exceptions, onionrusers
import json, os, sys, datetime, base64

class Block:
blockCacheOrder = list() # NEVER write your own code that writes to this!
blockCache = dict() # should never be accessed directly, look at Block.getCache()

def __init__(self, hash = None, core = None, type = None, content = None):
def __init__(self, hash = None, core = None, type = None, content = None, expire=None):
# take from arguments
# sometimes people input a bytes object instead of str in `hash`
if (not hash is None) and isinstance(hash, bytes):
@@ -35,6 +35,7 @@ class Block:
self.core = core
self.btype = type
self.bcontent = content
self.expire = expire

# initialize variables
self.valid = True
@@ -90,9 +91,18 @@ class Block:
self.signature = core._crypto.pubKeyDecrypt(self.signature, anonymous=anonymous, encodedData=encodedData)
self.signer = core._crypto.pubKeyDecrypt(self.signer, anonymous=anonymous, encodedData=encodedData)
self.signedData = json.dumps(self.bmetadata) + self.bcontent.decode()
try:
assert self.bmetadata['forwardEnc'] is True
except (AssertionError, KeyError) as e:
pass
else:
try:
self.bcontent = onionrusers.OnionrUser(self.getCore(), self.signer).forwardDecrypt(self.bcontent)
except (onionrexceptions.DecryptionError, nacl.exceptions.CryptoError) as e:
logger.error(str(e))
pass
except nacl.exceptions.CryptoError:
pass
#logger.debug('Could not decrypt block. Either invalid key or corrupted data')
logger.debug('Could not decrypt block. Either invalid key or corrupted data')
else:
retData = True
self.decrypted = True
@@ -149,7 +159,7 @@ class Block:

# read from file if it's still None
if blockdata is None:
filelocation = 'data/blocks/%s.dat' % self.getHash()
filelocation = self.core.dataDir + 'blocks/%s.dat' % self.getHash()

if readfile:
with open(filelocation, 'rb') as f:
@@ -177,6 +187,7 @@ class Block:
# signed data is jsonMeta + block content (no linebreak)
self.signedData = (None if not self.isSigned() else self.getHeader('meta') + self.getContent())
self.date = self.getCore().getBlockDate(self.getHash())
self.claimedTime = self.getHeader('time', None)

if not self.getDate() is None:
self.date = datetime.datetime.fromtimestamp(self.getDate())
@@ -226,7 +237,7 @@ class Block:
blockFile.write(self.getRaw().encode())
self.update()
else:
self.hash = self.getCore().insertBlock(self.getContent(), header = self.getType(), sign = sign)
self.hash = self.getCore().insertBlock(self.getContent(), header = self.getType(), sign = sign, expire=self.getExpire())
self.update()

return self.getHash()
@@ -239,6 +250,15 @@ class Block:

# getters

def getExpire(self):
'''
Returns the expire time for a block

Outputs:
- (int): the expire time for a block, or None
'''
return self.expire

def getHash(self):
'''
Returns the hash of the block if saved to file
@@ -726,7 +746,7 @@ class Block:
if type(hash) == Block:
blockfile = hash.getBlockFile()
else:
blockfile = 'data/blocks/%s.dat' % hash
blockfile = onionrcore.Core().dataDir + 'blocks/%s.dat' % hash

return os.path.exists(blockfile) and os.path.isfile(blockfile)



+ 49
- 0
onionr/onionrchat.py View File

@@ -0,0 +1,49 @@
'''
Onionr - P2P Anonymous Storage Network

Onionr Chat Messages
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import logger, time

class OnionrChat:
def __init__(self, communicatorInst):
'''OnionrChat uses onionrsockets (handled by the communicator) to exchange direct chat messages'''
self.communicator = communicatorInst
self._core = self.communicator._core
self._utils = self._core._utils

self.chats = {} # {'peer': {'date': date, message': message}}
self.chatSend = {}

def chatHandler(self):
while not self.communicator.shutdown:
for peer in self._core.socketServerConnData:
try:
assert self._core.socketReasons[peer] == "chat"
except (AssertionError, KeyError) as e:
logger.warn('Peer is not for chat')
continue
else:
self.chats[peer] = {'date': self._core.socketServerConnData[peer]['date'], 'data': self._core.socketServerConnData[peer]['data']}
logger.info("CHAT MESSAGE RECIEVED: %s" % self.chats[peer]['data'])
for peer in self.communicator.socketClient.sockets:
try:
logger.info(self.communicator.socketClient.connPool[peer]['data'])
self.communicator.socketClient.sendData(peer, "lol")
except:
pass
time.sleep(2)

+ 20
- 51
onionr/onionrcrypto.py View File

@@ -1,5 +1,5 @@
'''
Onionr - P2P Microblogging Platform & Social network
Onionr - P2P Anonymous Storage Network

This file handles Onionr's cryptography.
'''
@@ -24,51 +24,32 @@ if sys.version_info[0] == 3 and sys.version_info[1] < 6:
from dependencies import secrets
elif sys.version_info[0] == 3 and sys.version_info[1] >= 6:
import secrets
import config

class OnionrCrypto:
def __init__(self, coreInstance):
config.reload()
self._core = coreInstance
self._keyFile = 'data/keys.txt'
self.keyPowFile = 'data/keyPow.txt'
self._keyFile = self._core.dataDir + 'keys.txt'
self.pubKey = None
self.privKey = None

self.secrets = secrets

self.pubKeyPowToken = None
#self.pubKeyPowHash = None

self.HASH_ID_ROUNDS = 2000

# Load our own pub/priv Ed25519 keys, gen & save them if they don't exist
if os.path.exists(self._keyFile):
with open('data/keys.txt', 'r') as keys:
with open(self._core.dataDir + 'keys.txt', 'r') as keys:
keys = keys.read().split(',')
self.pubKey = keys[0]
self.privKey = keys[1]
try:
with open(self.keyPowFile, 'r') as powFile:
data = powFile.read()
self.pubKeyPowToken = data
except (FileNotFoundError, IndexError):
pass
else:
keys = self.generatePubKey()
self.pubKey = keys[0]
self.privKey = keys[1]
with open(self._keyFile, 'w') as keyfile:
keyfile.write(self.pubKey + ',' + self.privKey)
with open(self.keyPowFile, 'w') as keyPowFile:
proof = onionrproofs.DataPOW(self.pubKey)
logger.info('Doing necessary work to insert our public key')
while True:
time.sleep(0.2)
powToken = proof.getResult()
if powToken != False:
break
keyPowFile.write(base64.b64encode(powToken[1]).decode())
self.pubKeyPowToken = powToken[1]
self.pubKeyPowHash = powToken[0]
return

def edVerify(self, data, key, sig, encodedData=True):
@@ -76,7 +57,10 @@ class OnionrCrypto:
try:
key = nacl.signing.VerifyKey(key=key, encoder=nacl.encoding.Base32Encoder)
except nacl.exceptions.ValueError:
logger.warn('Signature by unknown key (cannot reverse hash)')
#logger.debug('Signature by unknown key (cannot reverse hash)')
return False
except binascii.Error:
logger.warn('Could not load key for verification, invalid padding')
return False
retData = False
sig = base64.b64decode(sig)
@@ -125,7 +109,7 @@ class OnionrCrypto:
encoding = nacl.encoding.RawEncoder

if self.privKey != None and not anonymous:
ownKey = nacl.signing.SigningKey(seed=self.privKey, encoder=nacl.encoding.Base32Encoder)
ownKey = nacl.signing.SigningKey(seed=self.privKey, encoder=nacl.encoding.Base32Encoder).to_curve25519_private_key()
key = nacl.signing.VerifyKey(key=pubkey, encoder=nacl.encoding.Base32Encoder).to_curve25519_public_key()
ourBox = nacl.public.Box(ownKey, key)
retVal = ourBox.encrypt(data.encode(), encoder=encoding)
@@ -139,9 +123,9 @@ class OnionrCrypto:
retVal = anonBox.encrypt(data, encoder=encoding)
return retVal

def pubKeyDecrypt(self, data, pubkey='', anonymous=False, encodedData=False):
def pubKeyDecrypt(self, data, pubkey='', privkey='', anonymous=False, encodedData=False):
'''pubkey decrypt (Curve25519, taken from Ed25519 pubkey)'''
retVal = False
decrypted = False
if encodedData:
encoding = nacl.encoding.Base64Encoder
else:
@@ -151,30 +135,14 @@ class OnionrCrypto:
ourBox = nacl.public.Box(ownKey, pubkey)
decrypted = ourBox.decrypt(data, encoder=encoding)
elif anonymous:
anonBox = nacl.public.SealedBox(ownKey)
if self._core._utils.validatePubKey(privkey):
privkey = nacl.signing.SigningKey(seed=privkey, encoder=nacl.encoding.Base32Encoder()).to_curve25519_private_key()
anonBox = nacl.public.SealedBox(privkey)
else:
anonBox = nacl.public.SealedBox(ownKey)
decrypted = anonBox.decrypt(data, encoder=encoding)
return decrypted

def symmetricPeerEncrypt(self, data, peer):
'''Salsa20 encrypt data to peer (with mac)
this function does not accept a key, it is a wrapper for encryption with a peer
'''
key = self._core.getPeerInfo(4)
if type(key) != bytes:
key = self._core.getPeerInfo(2)
encrypted = self.symmetricEncrypt(data, key, encodedKey=True)
return encrypted

def symmetricPeerDecrypt(self, data, peer):
'''Salsa20 decrypt data from peer (with mac)
this function does not accept a key, it is a wrapper for encryption with a peer
'''
key = self._core.getPeerInfo(4)
if type(key) != bytes:
key = self._core.getPeerInfo(2)
decrypted = self.symmetricDecrypt(data, key, encodedKey=True)
return decrypted

def symmetricEncrypt(self, data, key, encodedKey=False, returnEncoded=True):
'''Encrypt data to a 32-byte key (Salsa20-Poly1305 MAC)'''
if encodedKey:
@@ -282,7 +250,8 @@ class OnionrCrypto:
<