kev
/
Onionr
1
0
Fork 0
Code Issues 24 Pull Requests Projects Releases 2 Wiki Activity

Refactor Onionr

This commit is contained in:
Arinerron 2018-06-13 21:17:58 -07:00
parent 6cb69c7187
commit af237eab0b
11 changed files with 184 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
onionr/api.py
View File

@ -52,7 +52,7 @@ class API:
config.reload()
if config.get('devmode', True):
if config.get('dev_mode', True):
self._developmentMode = True
logger.set_level(logger.LEVEL_DEBUG)
else:
@ -65,12 +65,12 @@ class API:
self._crypto = onionrcrypto.OnionrCrypto(self._core)
self._utils = onionrutils.OnionrUtils(self._core)
app = flask.Flask(__name__)
bindPort = int(config.get('client')['port'])
bindPort = int(config.get('client.port', 59496))
self.bindPort = bindPort
self.clientToken = config.get('client')['client_hmac']
self.clientToken = config.get('client.hmac')
self.timeBypassToken = base64.b16encode(os.urandom(32)).decode()
self.i2pEnabled = config.get('i2p', {'host' : False})['host']
self.i2pEnabled = config.get('i2p.host', False)
self.mimeType = 'text/plain'
@ -82,7 +82,7 @@ class API:
self.host = '.'.join(hostOctets)
else:
self.host = '127.0.0.1'
with open('data/host.txt', 'w') as file:
file.write(self.host)

8
onionr/communicator.py
View File

@ -75,7 +75,7 @@ class OnionrCommunicate:
plugins.reload()
# Print nice header thing :)
if config.get('display_header', True):
if config.get('general.display_header', True):
self.header()
while True:
@ -149,7 +149,7 @@ class OnionrCommunicate:
logger.info('Checking for callbacks with connection %s...' % data['id'])
self.check_callbacks(data, config.get('dc_execcallbacks', True))
self.check_callbacks(data, config.get('general.dc_execcallbacks', True))
events.event('incoming_direct_connection', data = {'callback' : True, 'communicator' : self, 'data' : data})
except Exception as e:
@ -349,7 +349,7 @@ class OnionrCommunicate:
If yet another callback is requested, it can be put in the `callback` parameter.
'''
if config.get('dc_response', True):
if config.get('general.dc_response', True):
data['id'] = identifier
data['sender'] = open('data/hs/hostname').read()
data['callback'] = True
@ -768,7 +768,7 @@ class OnionrCommunicate:
shouldRun = False
debug = True
developmentMode = False
if config.get('devmode', True):
if config.get('general.dev_mode', True):
developmentMode = True
try:
if sys.argv[1] == 'run':

20
onionr/communicator2.py
View File

@ -37,7 +37,7 @@ class OnionrCommunicatorDaemon:
self.threadCounts = {}
self.shutdown = False
# Clear the daemon queue for any dead messages
if os.path.exists(self._core.queueDB):
self._core.clearDaemonQueue()
@ -46,7 +46,7 @@ class OnionrCommunicatorDaemon:
plugins.reload()
# Print nice header thing :)
if config.get('display_header', True):
if config.get('general.display_header', True):
self.header()
if debug or developmentMode:
@ -63,7 +63,7 @@ class OnionrCommunicatorDaemon:
i.processTimer()
time.sleep(self.delay)
logger.info('Goodbye.')
def decrementThreadCount(self, threadName):
if self.threadCounts[threadName] > 0:
self.threadCounts[threadName] -= 1
@ -102,8 +102,8 @@ class OnionrCommunicatorDaemon:
logger.debug('failed to connect to ' + address)
else:
logger.warn('Could not connect to any peer')
return retData
return retData
def heartbeat(self):
'''Show a heartbeat debug message'''
logger.debug('Communicator heartbeat')
@ -126,7 +126,7 @@ class OnionrCommunicatorDaemon:
else:
logger.info('Recieved daemonQueue command:' + cmd[0])
self.decrementThreadCount('daemonCommands')
def printOnlinePeers(self):
'''logs online peer list'''
if len(self.onlinePeers) == 0:
@ -148,7 +148,7 @@ class OnionrCommunicatorDaemon:
if announceCount == announceAmount:
logger.warn('Could not introduce node. Try again soon')
break
def peerAction(self, peer, action, data=''):
'''Perform a get request to a peer'''
logger.info('Performing ' + action + ' with ' + peer + ' on port ' + str(self.proxyPort))
@ -177,7 +177,7 @@ class OnionrCommunicatorDaemon:
# only to stdout, not file or log or anything
print(file.read().decode().replace('P', logger.colors.fg.pink).replace('W', logger.colors.reset + logger.colors.bold).replace('G', logger.colors.fg.green).replace('\n', logger.colors.reset + '\n'))
logger.info(logger.colors.fg.lightgreen + '-> ' + str(message) + logger.colors.reset + logger.colors.fg.lightgreen + ' <-\n')
class OnionrCommunicatorTimers:
def __init__(self, daemonInstance, timerFunction, frequency, makeThread=True, threadAmount=1, maxThreads=5):
self.timerFunction = timerFunction
@ -215,7 +215,7 @@ class OnionrCommunicatorTimers:
shouldRun = False
debug = True
developmentMode = False
if config.get('devmode', True):
if config.get('general.dev_mode', True):
developmentMode = True
try:
if sys.argv[1] == 'run':
@ -229,4 +229,4 @@ if shouldRun:
sys.exit(1)
pass
except Exception as e:
logger.error('Error occured in Communicator', error = e, timestamp = False)
logger.error('Error occured in Communicator', error = e, timestamp = False)

47
onionr/config.py
View File

@ -28,9 +28,20 @@ def get(key, default = None):
Gets the key from configuration, or returns `default`
'''
if is_set(key):
return get_config()[key]
return default
key = str(key).split('.')
data = _config
last = key.pop()
for item in key:
if (not item in data) or (not type(data[item]) == dict):
return default
data = data[item]
if not last in data:
return default
return data[last]
def set(key, value = None, savefile = False):
'''
@ -38,16 +49,40 @@ def set(key, value = None, savefile = False):
'''
global _config
key = str(key).split('.')
data = _config
last = key.pop()
for item in key:
if (not item in data) or (not type(data[item]) == dict):
data[item] = dict()
data = data[item]
if value is None:
del _config[key]
del data[last]
else:
_config[key] = value
data[last] = value
if savefile:
save()
def is_set(key):
return key in get_config() and not get_config()[key] is None
key = str(key).split('.')
data = _config
last = key.pop()
for item in key:
if (not item in data) or (not type(data[item]) == dict):
return False
data = data[item]
if not last in data:
return False
return True
def check():
'''

2
onionr/core.py
View File

@ -110,7 +110,7 @@ class Core:
'''
Add an address to the address database (only tor currently)
'''
if address == config.get('i2p', {'ownAddr' : None})['ownAddr']:
if address == config.get('i2p.ownAddr', None):
return False
if self._utils.validateID(address):

28
onionr/onionr.py
View File

@ -67,22 +67,22 @@ class Onionr:
config.set_config(json.loads(open('static-data/default_config.json').read())) # this is the default config, it will be overwritten if a config file already exists. Else, it saves it
else:
# the default config file doesn't exist, try hardcoded config
config.set_config({'devmode': True, 'log': {'file': {'output': True, 'path': 'data/output.log'}, 'console': {'output': True, 'color': True}}})
config.set_config({'dev_mode': True, 'log': {'file': {'output': True, 'path': 'data/output.log'}, 'console': {'output': True, 'color': True}}})
if not data_exists:
config.save()
config.reload() # this will read the configuration file into memory
settings = 0b000
if config.get('log', {'console': {'color': True}})['console']['color']:
if config.get('log.console.color', True):
settings = settings | logger.USE_ANSI
if config.get('log', {'console': {'output': True}})['console']['output']:
if config.get('log.console.output', True):
settings = settings | logger.OUTPUT_TO_CONSOLE
if config.get('log', {'file': {'output': True}})['file']['output']:
if config.get('log.file.output', True):
settings = settings | logger.OUTPUT_TO_FILE
logger.set_file(config.get('log', {'file': {'path': 'data/output.log'}})['file']['path'])
logger.set_file(config.get('log.file.path', '/tmp/onionr.log'))
logger.set_settings(settings)
if str(config.get('devmode', True)).lower() == 'true':
if str(config.get('general.dev_mode', True)).lower() == 'true':
self._developmentMode = True
logger.set_level(logger.LEVEL_DEBUG)
else:
@ -147,7 +147,7 @@ class Onionr:
randomPort = random.randint(1024, 65535)
if self.onionrUtils.checkPort(randomPort):
break
config.set('client', {'participate': 'true', 'client_hmac': base64.b16encode(os.urandom(32)).decode('utf-8'), 'port': randomPort, 'api_version': API_VERSION}, True)
config.set('client', {'participate': 'true', 'hmac': base64.b16encode(os.urandom(32)).decode('utf-8'), 'port': randomPort, 'api_version': API_VERSION}, True)
self.cmds = {
'': self.showHelpSuggestion,
@ -260,7 +260,7 @@ class Onionr:
self.onionrCore.daemonQueueAdd('connectedPeers')
def getWebPassword(self):
return config.get('client')['client_hmac']
return config.get('client.hmac')
def getHelp(self):
return self.cmdhelp
@ -532,12 +532,12 @@ class Onionr:
logger.info('Do ' + logger.colors.bold + sys.argv[0] + ' --help' + logger.colors.reset + logger.colors.fg.green + ' for Onionr help.')
def start(self, input = False):
def start(self, input = False, override = False):
'''
Starts the Onionr daemon
'''
if os.path.exists('.onionr-lock'):
if os.path.exists('.onionr-lock') and not override:
logger.fatal('Cannot start. Daemon is already running, or it did not exit cleanly.\n(if you are sure that there is not a daemon running, delete .onionr-lock & try again).')
else:
if not self.debug and not self._developmentMode:
@ -558,7 +558,7 @@ class Onionr:
if not os.environ.get("WERKZEUG_RUN_MAIN") == "true":
if self._developmentMode:
logger.warn('DEVELOPMENT MODE ENABLED (THIS IS LESS SECURE!)', timestamp = False)
net = NetController(config.get('client')['port'])
net = NetController(config.get('client.port', 59496))
logger.info('Tor is starting...')
if not net.startTor():
sys.exit(1)
@ -566,7 +566,7 @@ class Onionr:
logger.info('Our Public key: ' + self.onionrCore._crypto.pubKey)
time.sleep(1)
try:
if config.get('newCommunicator'):
if config.get('general.newCommunicator', False):
communicatorDaemon = './communicator2.py'
logger.info('Using new communicator')
except NameError:
@ -586,7 +586,7 @@ class Onionr:
logger.warn('Killing the running daemon...', timestamp = False)
try:
events.event('daemon_stop', onionr = self)
net = NetController(config.get('client')['port'])
net = NetController(config.get('client.port', 59496))
try:
self.onionrUtils.localCommand('shutdown')
except requests.exceptions.ConnectionError:
@ -625,7 +625,7 @@ class Onionr:
# count stats
'div2' : True,
'Known Peers Count' : str(len(self.onionrCore.listPeers()) - 1),
'Enabled Plugins Count' : str(len(config.get('plugins')['enabled'])) + ' / ' + str(len(os.listdir('data/plugins/'))),
'Enabled Plugins Count' : str(len(config.get('plugins.enabled', list()))) + ' / ' + str(len(os.listdir('data/plugins/'))),
'Known Blocks Count' : str(totalBlocks),
'Percent Blocks Signed' : str(round(100 * signedBlocks / max(totalBlocks, 1), 2)) + '%'
}

79
onionr/onionrblockapi.py
View File

@ -18,10 +18,13 @@
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import core as onionrcore, logger
import json, os, datetime, base64
import core as onionrcore, logger, config
import json, os, sys, datetime, base64
class Block:
blockCacheOrder = list() # NEVER write your own code that writes to this!
blockCache = dict() # should never be accessed directly, look at Block.getCache()
def __init__(self, hash = None, core = None):
'''
Initializes Onionr
@ -43,6 +46,9 @@ class Block:
self.hash = None
self.core = None
else:
if type(hash) == bytes:
hash = hash.decode()
self.btype = ''
self.bcontent = ''
self.hash = hash
@ -65,7 +71,8 @@ class Block:
if self.getCore() is None:
self.core = onionrcore.Core()
if not self.getHash() is None:
self.update()
if not self.update():
logger.debug('Failed to open block %s.' % self.getHash())
# logic
@ -93,16 +100,23 @@ class Block:
if blockdata is None:
filelocation = file
readfile = True
if filelocation is None:
if self.getHash() is None:
return False
try:
filelocation = 'data/blocks/%s.dat' % self.getHash().decode()
except AttributeError:
elif self.getHash() in Block.getCache():
# get the block from cache, if it's in it
blockdata = Block.getCache(self.getHash())
readfile = False
# read from file if it's still None
if blockdata is None:
filelocation = 'data/blocks/%s.dat' % self.getHash()
with open(filelocation, 'rb') as f:
blockdata = f.read().decode('utf-8')
if readfile:
with open(filelocation, 'rb') as f:
blockdata = f.read().decode()
self.blockFile = filelocation
else:
@ -126,6 +140,10 @@ class Block:
self.date = datetime.datetime.fromtimestamp(self.getDate())
self.valid = True
if len(self.getRaw()) <= config.get('allocations.blockCache', 500000):
self.cache()
return True
except Exception as e:
logger.error('Failed to update block data.', error = e, timestamp = False)
@ -641,11 +659,54 @@ class Block:
- (bool): whether or not the block file exists
'''
# no input data? scrap it.
if hash is None:
return False
elif type(hash) == Block:
if type(hash) == Block:
blockfile = hash.getBlockFile()
else:
blockfile = 'data/blocks/%s.dat' % hash
return os.path.exists(blockfile) and os.path.isfile(blockfile)
def getCache(hash = None):
# give a list of the hashes of the cached blocks
if hash is None:
return list(Block.blockCache.keys())
# if they inputted self or a Block, convert to hash
if type(hash) == Block:
hash = hash.getHash()
# just to make sure someone didn't put in a bool or something lol
hash = str(hash)
# if it exists, return its content
if hash in Block.getCache():
return Block.blockCache[hash]
return None
def cache(block, override = False):
# why even bother if they're giving bad data?
if not type(block) == Block:
return False
# only cache if written to file
if block.getHash() is None:
return False
# if it's already cached, what are we here for?
if block.getHash() in Block.getCache() and not override:
return False
# dump old cached blocks if the size exeeds the maximum
if sys.getsizeof(Block.blockCacheOrder) >= config.get('allocations.blockCacheTotal', 50000000): # 50MB default cache size
del Block.blockCache[blockCacheOrder.pop(0)]
# cache block content
Block.blockCache[block.getHash()] = block.getRaw()
Block.blockCacheOrder.append(block.getHash())
return True

11
onionr/onionrexceptions.py
View File

@ -18,7 +18,16 @@
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
# general exceptions
class NotFound(Exception):
pass
class Unknown(Exception):
pass
class Invalid(Exception):
pass
# network level exceptions
class MissingPort(Exception):
pass
class InvalidAddress(Exception):
pass
pass

10
onionr/onionrplugins.py
View File

@ -64,9 +64,7 @@ def enable(name, onionr = None, start_event = True):
enabled_plugins = get_enabled_plugins()
if not name in enabled_plugins:
enabled_plugins.append(name)
config_plugins = config.get('plugins')
config_plugins['enabled'] = enabled_plugins
config.set('plugins', config_plugins, True)
config.set('plugins.enabled', enabled_plugins, True)
events.call(get_plugin(name), 'enable', onionr)
@ -93,9 +91,7 @@ def disable(name, onionr = None, stop_event = True):
if is_enabled(name):
enabled_plugins = get_enabled_plugins()
enabled_plugins.remove(name)
config_plugins = config.get('plugins')
config_plugins['enabled'] = enabled_plugins
config.set('plugins', config_plugins, True)
config.set('plugins.enabled', enabled_plugins, True)
if exists(name):
events.call(get_plugin(name), 'disable', onionr)
@ -187,7 +183,7 @@ def get_enabled_plugins():
config.reload()
return config.get('plugins')['enabled']
return config.get('plugins.enabled', list())
def is_enabled(name):
'''

2
onionr/onionrutils.py
View File

@ -177,7 +177,7 @@ class OnionrUtils:
self.getTimeBypassToken()
# TODO: URL encode parameters, just as an extra measure. May not be needed, but should be added regardless.
try:
retData = requests.get('http://' + open('data/host.txt', 'r').read() + ':' + str(config.get('client')['port']) + '/client/?action=' + command + '&token=' + str(config.get('client')['client_hmac']) + '&timingToken=' + self.timingToken).text
retData = requests.get('http://%s:%s/client/?action=%s&token=%s&timingToken=' % (open('data/host.txt', 'r').read(), config.get('client.port', 59496), command, config.get('client.hmac'), self.timingToken)).text
except Exception as error:
if not silent:
logger.error('Failed to make local request (command: %s).' % command, error=error)

34
onionr/static-data/default_config.json
View File

@ -1,8 +1,18 @@
{
"devmode": true,
"dc_response": true,
"general" : {
"dev_mode": true,
"display_header" : true,
"newCommunicator": false,
"dc_response": true,
"dc_execcallbacks" : true
},
"client" : {
},
"display_header" : true,
"log": {
"file": {
"output": true,
@ -15,14 +25,20 @@
}
},
"tor" : {
},
"i2p":{
"host": false,
"connect": true,
"ownAddr": ""
"host": false,
"connect": true,
"ownAddr": ""
},
"allocations":{
"disk": 1000000000,
"netTotal": 1000000000
},
"newCommunicator": false
"netTotal": 1000000000,
"blockCache" : 5000000,
"blockCacheTotal" : 50000000
}
}