kev
/
Onionr
1
0
Fork 0
Code Issues 24 Pull Requests Projects Releases 2 Wiki Activity

work on sockets, added profile setter script

This commit is contained in:
Kevin Froman 2018-09-25 23:58:11 -05:00
parent 8b4105fac4
commit 1a856c365f
16 changed files with 140 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
onionr/api.py
View File

@ -102,7 +102,7 @@ class API:
self.mimeType = 'text/plain'
self.overrideCSP = False
with open('data/time-bypass.txt', 'w') as bypass:
with open(self._core.dataDir + 'time-bypass.txt', 'w') as bypass:
bypass.write(self.timeBypassToken)
if not debug and not self._developmentMode:
@ -111,7 +111,7 @@ class API:
else:
self.host = '127.0.0.1'
with open('data/host.txt', 'w') as file:
with open(self._core.dataDir + 'host.txt', 'w') as file:
file.write(self.host)
@app.before_request
@ -466,7 +466,7 @@ class API:
elif action == 'getData':
resp = ''
if self._utils.validateHash(data):
if os.path.exists('data/blocks/' + data + '.dat'):
if os.path.exists(self._core.dataDir + 'blocks/' + data + '.dat'):
block = Block(hash=data.encode(), core=self._core)
resp = base64.b64encode(block.getRaw().encode()).decode()
if len(resp) == 0:

0
onionr/blockprocessor.py
View File

2
onionr/communicator2.py
View File

@ -459,7 +459,7 @@ class OnionrCommunicatorDaemon:
self.announce(cmd[1])
elif cmd[0] == 'runCheck':
logger.debug('Status check; looks good.')
open('data/.runcheck', 'w+').close()
open(self._core.dataDir + '.runcheck', 'w+').close()
elif cmd[0] == 'connectedPeers':
self.printOnlinePeers()
elif cmd[0] == 'kex':

9
onionr/config.py
View File

@ -20,7 +20,14 @@
import os, json, logger
_configfile = os.path.abspath('data/config.json')
try:
dataDir = os.environ['ONIONR_HOME']
if not dataDir.endswith('/'):
dataDir += '/'
except KeyError:
dataDir = 'data/'
_configfile = os.path.abspath(dataDir + 'config.json')
_config = {}
def get(key, default = None):

44
onionr/core.py
View File

@ -35,20 +35,28 @@ class Core:
'''
Initialize Core Onionr library
'''
try:
self.queueDB = 'data/queue.db'
self.peerDB = 'data/peers.db'
self.blockDB = 'data/blocks.db'
self.blockDataLocation = 'data/blocks/'
self.addressDB = 'data/address.db'
self.dataDir = os.environ['ONIONR_HOME']
if not self.dataDir.endswith('/'):
self.dataDir += '/'
except KeyError:
self.dataDir = 'data/'
try:
self.queueDB = self.dataDir + 'queue.db'
self.peerDB = self.dataDir + 'peers.db'
self.blockDB = self.dataDir + 'blocks.db'
self.blockDataLocation = self.dataDir + 'blocks/'
self.addressDB = self.dataDir + 'address.db'
self.hsAddress = ''
self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt'
self.bootstrapList = []
self.requirements = onionrvalues.OnionrValues()
self.torPort = torPort
self.dataNonceFile = 'data/block-nonces.dat'
self.dataNonceFile = self.dataDir + 'block-nonces.dat'
self.dbCreate = dbcreator.DBCreator(self)
self.forwardKeysFile = 'data/forward-keys.db'
self.forwardKeysFile = self.dataDir + 'forward-keys.db'
# Socket data, defined here because of multithreading constraints with gevent
self.killSockets = False
@ -57,20 +65,20 @@ class Core:
self.socketReasons = {}
self.socketServerResponseData = {}
self.usageFile = 'data/disk-usage.txt'
self.usageFile = self.dataDir + 'disk-usage.txt'
self.config = config
self.maxBlockSize = 10000000 # max block size in bytes
if not os.path.exists('data/'):
os.mkdir('data/')
if not os.path.exists('data/blocks/'):
os.mkdir('data/blocks/')
if not os.path.exists(self.dataDir):
os.mkdir(self.dataDir)
if not os.path.exists(self.dataDir + 'blocks/'):
os.mkdir(self.dataDir + 'blocks/')
if not os.path.exists(self.blockDB):
self.createBlockDB()
if os.path.exists('data/hs/hostname'):
with open('data/hs/hostname', 'r') as hs:
if os.path.exists(self.dataDir + '/hs/hostname'):
with open(self.dataDir + '/hs/hostname', 'r') as hs:
self.hsAddress = hs.read().strip()
# Load bootstrap address list
@ -95,8 +103,8 @@ class Core:
def refreshFirstStartVars(self):
'''Hack to refresh some vars which may not be set on first start'''
if os.path.exists('data/hs/hostname'):
with open('data/hs/hostname', 'r') as hs:
if os.path.exists(self.dataDir + '/hs/hostname'):
with open(self.dataDir + '/hs/hostname', 'r') as hs:
self.hsAddress = hs.read().strip()
def addPeer(self, peerID, powID, name=''):
@ -136,7 +144,7 @@ class Core:
'''
Add an address to the address database (only tor currently)
'''
if address == config.get('i2p.ownAddr', None):
if address == config.get('i2p.ownAddr', None) or address == self.hsAddress:
return False
if self._utils.validateID(address):
@ -197,7 +205,7 @@ class Core:
c.execute('Delete from hashes where hash=?;', t)
conn.commit()
conn.close()
blockFile = 'data/blocks/' + block + '.dat'
blockFile = self.dataDir + '/blocks/' + block + '.dat'
dataSize = 0
try:
''' Get size of data when loaded as an object/var, rather than on disk,

21
onionr/netcontroller.py
View File

@ -28,7 +28,14 @@ class NetController:
'''
def __init__(self, hsPort):
self.torConfigLocation = 'data/torrc'
try:
self.dataDir = os.environ['ONIONR_HOME']
if not self.dataDir.endswith('/'):
self.dataDir += '/'
except KeyError:
self.dataDir = 'data/'
self.torConfigLocation = self.dataDir + 'torrc'
self.readyState = False
self.socksPort = random.randint(1024, 65535)
self.hsPort = hsPort
@ -81,10 +88,10 @@ class NetController:
break
torrcData = '''SocksPort ''' + str(self.socksPort) + '''
HiddenServiceDir data/hs/
HiddenServiceDir ''' + self.dataDir + '''hs/
\n''' + hsVer + '''\n
HiddenServicePort 80 127.0.0.1:''' + str(self.hsPort) + '''
DataDirectory data/tordata/
DataDirectory ''' + self.dataDir + '''tordata/
CookieAuthentication 1
ControlPort ''' + str(controlPort) + '''
HashedControlPassword ''' + str(password) + '''
@ -140,11 +147,11 @@ HashedControlPassword ''' + str(password) + '''
logger.debug('Finished starting Tor.', timestamp=True)
self.readyState = True
myID = open('data/hs/hostname', 'r')
myID = open(self.dataDir + 'hs/hostname', 'r')
self.myID = myID.read().replace('\n', '')
myID.close()
torPidFile = open('data/torPid.txt', 'w')
torPidFile = open(self.dataDir + 'torPid.txt', 'w')
torPidFile.write(str(tor.pid))
torPidFile.close()
@ -156,7 +163,7 @@ HashedControlPassword ''' + str(password) + '''
'''
try:
pid = open('data/torPid.txt', 'r')
pid = open(self.dataDir + 'torPid.txt', 'r')
pidN = pid.read()
pid.close()
except FileNotFoundError:
@ -169,7 +176,7 @@ HashedControlPassword ''' + str(password) + '''
try:
os.kill(int(pidN), signal.SIGTERM)
os.remove('data/torPid.txt')
os.remove(self.dataDir + 'torPid.txt')
except ProcessLookupError:
pass
except FileNotFoundError:

35
onionr/onionr.py
View File

@ -57,18 +57,25 @@ class Onionr:
except FileNotFoundError:
pass
try:
self.dataDir = os.environ['ONIONR_HOME']
if not self.dataDir.endswith('/'):
self.dataDir += '/'
except KeyError:
self.dataDir = 'data/'
# Load global configuration data
data_exists = os.path.exists('data/')
data_exists = os.path.exists(self.dataDir)
if not data_exists:
os.mkdir('data/')
os.mkdir(self.dataDir)
if os.path.exists('static-data/default_config.json'):
config.set_config(json.loads(open('static-data/default_config.json').read())) # this is the default config, it will be overwritten if a config file already exists. Else, it saves it
else:
# the default config file doesn't exist, try hardcoded config
config.set_config({'dev_mode': True, 'log': {'file': {'output': True, 'path': 'data/output.log'}, 'console': {'output': True, 'color': True}}})
config.set_config({'dev_mode': True, 'log': {'file': {'output': True, 'path': self.dataDir + 'output.log'}, 'console': {'output': True, 'color': True}}})
if not data_exists:
config.save()
config.reload() # this will read the configuration file into memory
@ -80,7 +87,7 @@ class Onionr:
settings = settings | logger.OUTPUT_TO_CONSOLE
if config.get('log.file.output', True):
settings = settings | logger.OUTPUT_TO_FILE
logger.set_file(config.get('log.file.path', '/tmp/onionr.log'))
logger.set_file(config.get('log.file.path', '/tmp/onionr.log').replace('data/', self.dataDir))
logger.set_settings(settings)
if str(config.get('general.dev_mode', True)).lower() == 'true':
@ -102,15 +109,15 @@ class Onionr:
print('Enter password to decrypt:')
password = getpass.getpass()
result = self.onionrCore.dataDirDecrypt(password)
if os.path.exists('data/'):
if os.path.exists(self.dataDir):
break
else:
logger.error('Failed to decrypt: ' + result[1], timestamp = False)
else:
# If data folder does not exist
if not data_exists:
if not os.path.exists('data/blocks/'):
os.mkdir('data/blocks/')
if not os.path.exists(self.dataDir + 'blocks/'):
os.mkdir(self.dataDir + 'blocks/')
# Copy default plugins into plugins folder
if not os.path.exists(plugins.get_plugins_folder()):
@ -262,7 +269,7 @@ class Onionr:
if not self._developmentMode:
encryptionPassword = self.onionrUtils.getPassword('Enter password to encrypt directory: ')
self.onionrCore.dataDirEncrypt(encryptionPassword)
shutil.rmtree('data/')
shutil.rmtree(self.dataDir)
return
@ -695,7 +702,7 @@ class Onionr:
powToken = self.onionrCore._crypto.pubKeyPowToken
messages = {
# info about local client
'Onionr Daemon Status' : ((logger.colors.fg.green + 'Online') if self.onionrUtils.isCommunicatorRunning(timeout = 2) else logger.colors.fg.red + 'Offline'),
'Onionr Daemon Status' : ((logger.colors.fg.green + 'Online') if self.onionrUtils.isCommunicatorRunning(timeout = 9) else logger.colors.fg.red + 'Offline'),
'Public Key' : self.onionrCore._crypto.pubKey,
'POW Token' : powToken,
'Combined' : self.onionrCore._crypto.pubKey + '-' + powToken,
@ -704,14 +711,14 @@ class Onionr:
# file and folder size stats
'div1' : True, # this creates a solid line across the screen, a div
'Total Block Size' : onionrutils.humanSize(onionrutils.size('data/blocks/')),
'Total Plugin Size' : onionrutils.humanSize(onionrutils.size('data/plugins/')),
'Log File Size' : onionrutils.humanSize(onionrutils.size('data/output.log')),
'Total Block Size' : onionrutils.humanSize(onionrutils.size(self.dataDir + 'blocks/')),
'Total Plugin Size' : onionrutils.humanSize(onionrutils.size(self.dataDir + 'plugins/')),
'Log File Size' : onionrutils.humanSize(onionrutils.size(self.dataDir + 'output.log')),
# count stats
'div2' : True,
'Known Peers Count' : str(len(self.onionrCore.listPeers()) - 1),
'Enabled Plugins Count' : str(len(config.get('plugins.enabled', list()))) + ' / ' + str(len(os.listdir('data/plugins/'))),
'Enabled Plugins Count' : str(len(config.get('plugins.enabled', list()))) + ' / ' + str(len(os.listdir(self.dataDir + 'plugins/'))),
'Known Blocks Count' : str(totalBlocks),
'Percent Blocks Signed' : str(round(100 * signedBlocks / max(totalBlocks, 1), 2)) + '%'
}
@ -777,7 +784,7 @@ class Onionr:
def get_hostname(self):
try:
with open('./data/hs/hostname', 'r') as hostname:
with open('./' + self.dataDir + 'hs/hostname', 'r') as hostname:
return hostname.read().strip()
except Exception:
return None

2
onionr/onionrblacklist.py
View File

@ -20,7 +20,7 @@
import sqlite3, os, logger
class OnionrBlackList:
def __init__(self, coreInst):
self.blacklistDB = 'data/blacklist.db'
self.blacklistDB = coreInst.dataDir + 'blacklist.db'
self._core = coreInst
if not os.path.exists(self.blacklistDB):

4
onionr/onionrblockapi.py
View File

@ -149,7 +149,7 @@ class Block:
# read from file if it's still None
if blockdata is None:
filelocation = 'data/blocks/%s.dat' % self.getHash()
filelocation = self.core.dataDir + 'blocks/%s.dat' % self.getHash()
if readfile:
with open(filelocation, 'rb') as f:
@ -727,7 +727,7 @@ class Block:
if type(hash) == Block:
blockfile = hash.getBlockFile()
else:
blockfile = 'data/blocks/%s.dat' % hash
blockfile = onionrcore.Core().dataDir + 'blocks/%s.dat' % hash
return os.path.exists(blockfile) and os.path.isfile(blockfile)

9
onionr/onionrchat.py
View File

@ -27,6 +27,7 @@ class OnionrChat:
self._utils = self._core._utils
self.chats = {} # {'peer': {'date': date, message': message}}
self.chatSend = {}
def chatHandler(self):
while not self.communicator.shutdown:
@ -34,7 +35,15 @@ class OnionrChat:
try:
assert self._core.socketReasons[peer] == "chat"
except (AssertionError, KeyError) as e:
logger.warn('Peer is not for chat')
continue
else:
self.chats[peer] = {'date': self._core.socketServerConnData[peer]['date'], 'data': self._core.socketServerConnData[peer]['data']}
logger.info("CHAT MESSAGE RECIEVED: %s" % self.chats[peer]['data'])
for peer in self.communicator.socketClient.sockets:
try:
logger.info(self.communicator.socketClient.connPool[peer]['data'])
self.communicator.socketClient.sendData(peer, "lol")
except:
pass
time.sleep(2)

6
onionr/onionrcrypto.py
View File

@ -28,8 +28,8 @@ elif sys.version_info[0] == 3 and sys.version_info[1] >= 6:
class OnionrCrypto:
def __init__(self, coreInstance):
self._core = coreInstance
self._keyFile = 'data/keys.txt'
self.keyPowFile = 'data/keyPow.txt'
self._keyFile = self._core.dataDir + 'keys.txt'
self.keyPowFile = self._core.dataDir + 'keyPow.txt'
self.pubKey = None
self.privKey = None
@ -42,7 +42,7 @@ class OnionrCrypto:
# Load our own pub/priv Ed25519 keys, gen & save them if they don't exist
if os.path.exists(self._keyFile):
with open('data/keys.txt', 'r') as keys:
with open(self._core.dataDir + 'keys.txt', 'r') as keys:
keys = keys.read().split(',')
self.pubKey = keys[0]
self.privKey = keys[1]

11
onionr/onionrplugins.py
View File

@ -21,7 +21,14 @@
import os, re, importlib, config, logger
import onionrevents as events
_pluginsfolder = 'data/plugins/'
try:
dataDir = os.environ['ONIONR_HOME']
if not dataDir.endswith('/'):
dataDir += '/'
except KeyError:
dataDir = 'data/'
_pluginsfolder = dataDir + 'plugins/'
_instances = dict()
def reload(onionr = None, stop_event = True):
@ -217,7 +224,7 @@ def get_plugin_data_folder(name, absolute = True):
Returns the location of a plugin's data folder
'''
return get_plugins_folder(name, absolute) + 'data/'
return get_plugins_folder(name, absolute) + dataDir
def check():
'''

28
onionr/onionrsockets.py
View File

@ -68,8 +68,8 @@ class OnionrSocketServer:
retData = self._core.socketServerResponseData[myPeer]
except KeyError:
pass
self._core.socketServerConnData[myPeer] = ''
else:
self._core.socketServerResponseData[myPeer] = ''
return retData
@ -99,11 +99,11 @@ class OnionrSocketServer:
controller.authenticate(config.get('tor.controlpassword'))
socket = controller.create_ephemeral_hidden_service({80: bindPort}, await_publication = True)
self.sockets[peer] = socket.service_id
self.sockets[peer] = socket.service_id + '.onion'
self.responseData[socket.service_id] = ''
self.responseData[socket.service_id + '.onion'] = ''
self._core.insertBlock(str(uuid.uuid4()), header='socket', sign=True, encryptType='asym', asymPeer=peer, meta={'reason': reason})
self._core.insertBlock(str(uuid.uuid4()), header='socket', sign=True, encryptType='asym', asymPeer=peer, meta={'reason': reason, 'address': socket.service_id + '.onion'})
self._core.socketReasons[peer] = reason
return
@ -123,16 +123,26 @@ class OnionrSocketClient:
logger.info('Trying to find socket server for %s' % (peer,))
# Find the newest open socket for a given peer
for block in self._core.getBlocksByType('socket'):
block = onionrblockapi.Block(block, core=self._myCore)
block = onionrblockapi.Block(block, core=self._core)
if block.decrypt():
if block.verifySig() and block.signer == peer:
theSigner = block.signer
try:
theSigner = theSigner.decode()
except AttributeError:
pass
if block.verifySig() and theSigner == peer:
address = block.getMetadata('address')
if self._core._utils.validateID(address):
# If we got their address, it is valid, and verified, we can break out
if block.getMetadata('reason') == 'chat':
if block.getMetadata('reason') == reason:
break
else:
logger.error('The socket the peer opened is not for %s' % (reason,))
else:
logger.error('Peer transport id is invalid for socket: %s' % (address,))
address = ''
else:
logger.warn('Block has invalid sig or id, was for %s' % (theSigner,))
if address != '':
logger.info('%s socket client started with %s' % (reason, peer))
self.sockets[peer] = address
@ -140,12 +150,14 @@ class OnionrSocketClient:
while not self.killSocket:
try:
data = self.sendData[peer]
logger.info('Sending %s to %s' % (data, peer))
except KeyError:
pass
else:
self.sendData[peer] = ''
postData = {'data': data}
self.connPool[peer] = {'date': self._core._utils.getEpoch(), 'data': self._core._utils.doPostRequest('http://' + address + '/dc/', data=postData)}
time.sleep(2)
def getResponse(self, peer):
retData = ''

15
onionr/onionrutils.py
View File

@ -52,8 +52,8 @@ class OnionrUtils:
Load our timingToken from disk for faster local HTTP API
'''
try:
if os.path.exists('data/time-bypass.txt'):
with open('data/time-bypass.txt', 'r') as bypass:
if os.path.exists(self._core.dataDir + 'time-bypass.txt'):
with open(self._core.dataDir + 'time-bypass.txt', 'r') as bypass:
self.timingToken = bypass.read()
except Exception as error:
logger.error('Failed to fetch time bypass token.', error = error)
@ -143,7 +143,7 @@ class OnionrUtils:
def getMyAddress(self):
try:
with open('./data/hs/hostname', 'r') as hostname:
with open('./' + self._core.dataDir + 'hs/hostname', 'r') as hostname:
return hostname.read().strip()
except Exception as error:
logger.error('Failed to read my address.', error = error)
@ -158,7 +158,7 @@ class OnionrUtils:
self.getTimeBypassToken()
# TODO: URL encode parameters, just as an extra measure. May not be needed, but should be added regardless.
try:
with open('data/host.txt', 'r') as host:
with open(self._core.dataDir + 'host.txt', 'r') as host:
hostname = host.read()
except FileNotFoundError:
return False
@ -270,9 +270,10 @@ class OnionrUtils:
if len(blockType) <= 10:
self._core.updateBlockInfo(blockHash, 'dataType', blockType)
onionrevents.event('processBlocks', data = {'block': myBlock, 'type': blockType, 'signer': signer, 'validSig': valid}, onionr = None)
onionrevents.event('processblocks', data = {'block': myBlock, 'type': blockType, 'signer': signer, 'validSig': valid}, onionr = None)
except TypeError:
logger.warn("Missing block information")
pass
def escapeAnsi(self, line):
@ -479,7 +480,7 @@ class OnionrUtils:
def isCommunicatorRunning(self, timeout = 5, interval = 0.1):
try:
runcheck_file = 'data/.runcheck'
runcheck_file = self._core.dataDir + '.runcheck'
if os.path.isfile(runcheck_file):
os.remove(runcheck_file)
@ -593,7 +594,7 @@ class OnionrUtils:
except ValueError as e:
logger.debug('Failed to make request', error = e)
except requests.exceptions.RequestException as e:
if not 'ConnectTimeoutError' in str(e):
if not 'ConnectTimeoutError' in str(e) and not 'Request rejected or failed' in str(e):
logger.debug('Error: %s' % str(e))
retData = False
return retData

11
onionr/static-data/default-plugins/metadataprocessor/main.py
View File

@ -58,7 +58,7 @@ def _processForwardKey(api, myBlock):
else:
raise onionrexceptions.InvalidPubkey("%s is nota valid pubkey key" % (key,))
def on_processBlocks(api):
def on_processblocks(api):
# Generally fired by utils.
myBlock = api.data['block']
blockType = api.data['type']
@ -77,21 +77,24 @@ def on_processBlocks(api):
# socket blocks
elif blockType == 'socket':
if api.data['validSig'] == True and myBlock.decrypted: # we check if it is decrypted as a way of seeing if it was for us
logger.info('Detected socket advertised to us...')
try:
address = api.data['address']
address = myBlock.getMetadata('address')
except KeyError:
raise onionrexceptions.MissingAddress("Missing address for new socket")
try:
port = api.data['port']
port = myBlock.getMetadata('port')
except KeyError:
raise ValueError("Missing port for new socket")
try:
reason = api.data['reason']
reason = myBlock.getMetadata('reason')
except KeyError:
raise ValueError("Missing socket reason")
socketInfo = json.dumps({'peer': api.data['signer'], 'address': address, 'port': port, 'create': False, 'reason': reason})
api.get_core().daemonQueueAdd('addSocket', socketInfo)
else:
logger.warn("socket is not for us or is invalid")
def on_init(api, data = None):

7
setprofile.sh Executable file
View File

@ -0,0 +1,7 @@
#!/bin/bash
ONIONR_HOME=.
if [ $# -gt 0 ]; then
ONIONR_HOME=$1
export ONIONR_HOME
echo "set ONIONR_HOME to $ONIONR_HOME"
fi