diff --git a/onionr/api.py b/onionr/api.py index 6cb94104..c5493a80 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -52,7 +52,7 @@ class API: config.reload() - if config.get('devmode', True): + if config.get('dev_mode', True): self._developmentMode = True logger.set_level(logger.LEVEL_DEBUG) else: @@ -65,12 +65,12 @@ class API: self._crypto = onionrcrypto.OnionrCrypto(self._core) self._utils = onionrutils.OnionrUtils(self._core) app = flask.Flask(__name__) - bindPort = int(config.get('client')['port']) + bindPort = int(config.get('client.port', 59496)) self.bindPort = bindPort - self.clientToken = config.get('client')['client_hmac'] + self.clientToken = config.get('client.hmac') self.timeBypassToken = base64.b16encode(os.urandom(32)).decode() - self.i2pEnabled = config.get('i2p', {'host' : False})['host'] + self.i2pEnabled = config.get('i2p.host', False) self.mimeType = 'text/plain' @@ -82,7 +82,7 @@ class API: self.host = '.'.join(hostOctets) else: self.host = '127.0.0.1' - + with open('data/host.txt', 'w') as file: file.write(self.host) diff --git a/onionr/communicator.py b/onionr/communicator.py index 894a550f..82d5a3c2 100755 --- a/onionr/communicator.py +++ b/onionr/communicator.py @@ -75,7 +75,7 @@ class OnionrCommunicate: plugins.reload() # Print nice header thing :) - if config.get('display_header', True): + if config.get('general.display_header', True): self.header() while True: @@ -149,7 +149,7 @@ class OnionrCommunicate: logger.info('Checking for callbacks with connection %s...' % data['id']) - self.check_callbacks(data, config.get('dc_execcallbacks', True)) + self.check_callbacks(data, config.get('general.dc_execcallbacks', True)) events.event('incoming_direct_connection', data = {'callback' : True, 'communicator' : self, 'data' : data}) except Exception as e: @@ -349,7 +349,7 @@ class OnionrCommunicate: If yet another callback is requested, it can be put in the `callback` parameter. ''' - if config.get('dc_response', True): + if config.get('general.dc_response', True): data['id'] = identifier data['sender'] = open('data/hs/hostname').read() data['callback'] = True @@ -768,7 +768,7 @@ class OnionrCommunicate: shouldRun = False debug = True developmentMode = False -if config.get('devmode', True): +if config.get('general.dev_mode', True): developmentMode = True try: if sys.argv[1] == 'run': diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 54698ce4..2b5577c3 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -37,7 +37,7 @@ class OnionrCommunicatorDaemon: self.threadCounts = {} self.shutdown = False - + # Clear the daemon queue for any dead messages if os.path.exists(self._core.queueDB): self._core.clearDaemonQueue() @@ -46,7 +46,7 @@ class OnionrCommunicatorDaemon: plugins.reload() # Print nice header thing :) - if config.get('display_header', True): + if config.get('general.display_header', True): self.header() if debug or developmentMode: @@ -63,7 +63,7 @@ class OnionrCommunicatorDaemon: i.processTimer() time.sleep(self.delay) logger.info('Goodbye.') - + def decrementThreadCount(self, threadName): if self.threadCounts[threadName] > 0: self.threadCounts[threadName] -= 1 @@ -102,8 +102,8 @@ class OnionrCommunicatorDaemon: logger.debug('failed to connect to ' + address) else: logger.warn('Could not connect to any peer') - return retData - + return retData + def heartbeat(self): '''Show a heartbeat debug message''' logger.debug('Communicator heartbeat') @@ -126,7 +126,7 @@ class OnionrCommunicatorDaemon: else: logger.info('Recieved daemonQueue command:' + cmd[0]) self.decrementThreadCount('daemonCommands') - + def printOnlinePeers(self): '''logs online peer list''' if len(self.onlinePeers) == 0: @@ -148,7 +148,7 @@ class OnionrCommunicatorDaemon: if announceCount == announceAmount: logger.warn('Could not introduce node. Try again soon') break - + def peerAction(self, peer, action, data=''): '''Perform a get request to a peer''' logger.info('Performing ' + action + ' with ' + peer + ' on port ' + str(self.proxyPort)) @@ -177,7 +177,7 @@ class OnionrCommunicatorDaemon: # only to stdout, not file or log or anything print(file.read().decode().replace('P', logger.colors.fg.pink).replace('W', logger.colors.reset + logger.colors.bold).replace('G', logger.colors.fg.green).replace('\n', logger.colors.reset + '\n')) logger.info(logger.colors.fg.lightgreen + '-> ' + str(message) + logger.colors.reset + logger.colors.fg.lightgreen + ' <-\n') - + class OnionrCommunicatorTimers: def __init__(self, daemonInstance, timerFunction, frequency, makeThread=True, threadAmount=1, maxThreads=5): self.timerFunction = timerFunction @@ -215,7 +215,7 @@ class OnionrCommunicatorTimers: shouldRun = False debug = True developmentMode = False -if config.get('devmode', True): +if config.get('general.dev_mode', True): developmentMode = True try: if sys.argv[1] == 'run': @@ -229,4 +229,4 @@ if shouldRun: sys.exit(1) pass except Exception as e: - logger.error('Error occured in Communicator', error = e, timestamp = False) \ No newline at end of file + logger.error('Error occured in Communicator', error = e, timestamp = False) diff --git a/onionr/config.py b/onionr/config.py index fb8ac161..b18ba552 100644 --- a/onionr/config.py +++ b/onionr/config.py @@ -28,9 +28,20 @@ def get(key, default = None): Gets the key from configuration, or returns `default` ''' - if is_set(key): - return get_config()[key] - return default + key = str(key).split('.') + data = _config + + last = key.pop() + + for item in key: + if (not item in data) or (not type(data[item]) == dict): + return default + data = data[item] + + if not last in data: + return default + + return data[last] def set(key, value = None, savefile = False): ''' @@ -38,16 +49,40 @@ def set(key, value = None, savefile = False): ''' global _config + + key = str(key).split('.') + data = _config + + last = key.pop() + + for item in key: + if (not item in data) or (not type(data[item]) == dict): + data[item] = dict() + data = data[item] + if value is None: - del _config[key] + del data[last] else: - _config[key] = value + data[last] = value if savefile: save() def is_set(key): - return key in get_config() and not get_config()[key] is None + key = str(key).split('.') + data = _config + + last = key.pop() + + for item in key: + if (not item in data) or (not type(data[item]) == dict): + return False + data = data[item] + + if not last in data: + return False + + return True def check(): ''' diff --git a/onionr/core.py b/onionr/core.py index ef133bec..141c0480 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -110,7 +110,7 @@ class Core: ''' Add an address to the address database (only tor currently) ''' - if address == config.get('i2p', {'ownAddr' : None})['ownAddr']: + if address == config.get('i2p.ownAddr', None): return False if self._utils.validateID(address): diff --git a/onionr/onionr.py b/onionr/onionr.py index db2825a7..b0596166 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -67,22 +67,22 @@ class Onionr: config.set_config(json.loads(open('static-data/default_config.json').read())) # this is the default config, it will be overwritten if a config file already exists. Else, it saves it else: # the default config file doesn't exist, try hardcoded config - config.set_config({'devmode': True, 'log': {'file': {'output': True, 'path': 'data/output.log'}, 'console': {'output': True, 'color': True}}}) + config.set_config({'dev_mode': True, 'log': {'file': {'output': True, 'path': 'data/output.log'}, 'console': {'output': True, 'color': True}}}) if not data_exists: config.save() config.reload() # this will read the configuration file into memory settings = 0b000 - if config.get('log', {'console': {'color': True}})['console']['color']: + if config.get('log.console.color', True): settings = settings | logger.USE_ANSI - if config.get('log', {'console': {'output': True}})['console']['output']: + if config.get('log.console.output', True): settings = settings | logger.OUTPUT_TO_CONSOLE - if config.get('log', {'file': {'output': True}})['file']['output']: + if config.get('log.file.output', True): settings = settings | logger.OUTPUT_TO_FILE - logger.set_file(config.get('log', {'file': {'path': 'data/output.log'}})['file']['path']) + logger.set_file(config.get('log.file.path', '/tmp/onionr.log')) logger.set_settings(settings) - if str(config.get('devmode', True)).lower() == 'true': + if str(config.get('general.dev_mode', True)).lower() == 'true': self._developmentMode = True logger.set_level(logger.LEVEL_DEBUG) else: @@ -147,7 +147,7 @@ class Onionr: randomPort = random.randint(1024, 65535) if self.onionrUtils.checkPort(randomPort): break - config.set('client', {'participate': 'true', 'client_hmac': base64.b16encode(os.urandom(32)).decode('utf-8'), 'port': randomPort, 'api_version': API_VERSION}, True) + config.set('client', {'participate': 'true', 'hmac': base64.b16encode(os.urandom(32)).decode('utf-8'), 'port': randomPort, 'api_version': API_VERSION}, True) self.cmds = { '': self.showHelpSuggestion, @@ -260,7 +260,7 @@ class Onionr: self.onionrCore.daemonQueueAdd('connectedPeers') def getWebPassword(self): - return config.get('client')['client_hmac'] + return config.get('client.hmac') def getHelp(self): return self.cmdhelp @@ -532,12 +532,12 @@ class Onionr: logger.info('Do ' + logger.colors.bold + sys.argv[0] + ' --help' + logger.colors.reset + logger.colors.fg.green + ' for Onionr help.') - def start(self, input = False): + def start(self, input = False, override = False): ''' Starts the Onionr daemon ''' - if os.path.exists('.onionr-lock'): + if os.path.exists('.onionr-lock') and not override: logger.fatal('Cannot start. Daemon is already running, or it did not exit cleanly.\n(if you are sure that there is not a daemon running, delete .onionr-lock & try again).') else: if not self.debug and not self._developmentMode: @@ -558,7 +558,7 @@ class Onionr: if not os.environ.get("WERKZEUG_RUN_MAIN") == "true": if self._developmentMode: logger.warn('DEVELOPMENT MODE ENABLED (THIS IS LESS SECURE!)', timestamp = False) - net = NetController(config.get('client')['port']) + net = NetController(config.get('client.port', 59496)) logger.info('Tor is starting...') if not net.startTor(): sys.exit(1) @@ -566,7 +566,7 @@ class Onionr: logger.info('Our Public key: ' + self.onionrCore._crypto.pubKey) time.sleep(1) try: - if config.get('newCommunicator'): + if config.get('general.newCommunicator', False): communicatorDaemon = './communicator2.py' logger.info('Using new communicator') except NameError: @@ -586,7 +586,7 @@ class Onionr: logger.warn('Killing the running daemon...', timestamp = False) try: events.event('daemon_stop', onionr = self) - net = NetController(config.get('client')['port']) + net = NetController(config.get('client.port', 59496)) try: self.onionrUtils.localCommand('shutdown') except requests.exceptions.ConnectionError: @@ -625,7 +625,7 @@ class Onionr: # count stats 'div2' : True, 'Known Peers Count' : str(len(self.onionrCore.listPeers()) - 1), - 'Enabled Plugins Count' : str(len(config.get('plugins')['enabled'])) + ' / ' + str(len(os.listdir('data/plugins/'))), + 'Enabled Plugins Count' : str(len(config.get('plugins.enabled', list()))) + ' / ' + str(len(os.listdir('data/plugins/'))), 'Known Blocks Count' : str(totalBlocks), 'Percent Blocks Signed' : str(round(100 * signedBlocks / max(totalBlocks, 1), 2)) + '%' } diff --git a/onionr/onionrblockapi.py b/onionr/onionrblockapi.py index 335d6d23..cc50416a 100644 --- a/onionr/onionrblockapi.py +++ b/onionr/onionrblockapi.py @@ -18,10 +18,13 @@ along with this program. If not, see . ''' -import core as onionrcore, logger -import json, os, datetime, base64 +import core as onionrcore, logger, config +import json, os, sys, datetime, base64 class Block: + blockCacheOrder = list() # NEVER write your own code that writes to this! + blockCache = dict() # should never be accessed directly, look at Block.getCache() + def __init__(self, hash = None, core = None): ''' Initializes Onionr @@ -43,6 +46,9 @@ class Block: self.hash = None self.core = None else: + if type(hash) == bytes: + hash = hash.decode() + self.btype = '' self.bcontent = '' self.hash = hash @@ -65,7 +71,8 @@ class Block: if self.getCore() is None: self.core = onionrcore.Core() if not self.getHash() is None: - self.update() + if not self.update(): + logger.debug('Failed to open block %s.' % self.getHash()) # logic @@ -93,16 +100,23 @@ class Block: if blockdata is None: filelocation = file + readfile = True + if filelocation is None: if self.getHash() is None: return False - try: - filelocation = 'data/blocks/%s.dat' % self.getHash().decode() - except AttributeError: + elif self.getHash() in Block.getCache(): + # get the block from cache, if it's in it + blockdata = Block.getCache(self.getHash()) + readfile = False + + # read from file if it's still None + if blockdata is None: filelocation = 'data/blocks/%s.dat' % self.getHash() - with open(filelocation, 'rb') as f: - blockdata = f.read().decode('utf-8') + if readfile: + with open(filelocation, 'rb') as f: + blockdata = f.read().decode() self.blockFile = filelocation else: @@ -126,6 +140,10 @@ class Block: self.date = datetime.datetime.fromtimestamp(self.getDate()) self.valid = True + + if len(self.getRaw()) <= config.get('allocations.blockCache', 500000): + self.cache() + return True except Exception as e: logger.error('Failed to update block data.', error = e, timestamp = False) @@ -641,11 +659,54 @@ class Block: - (bool): whether or not the block file exists ''' + # no input data? scrap it. if hash is None: return False - elif type(hash) == Block: + + if type(hash) == Block: blockfile = hash.getBlockFile() else: blockfile = 'data/blocks/%s.dat' % hash return os.path.exists(blockfile) and os.path.isfile(blockfile) + + def getCache(hash = None): + # give a list of the hashes of the cached blocks + if hash is None: + return list(Block.blockCache.keys()) + + # if they inputted self or a Block, convert to hash + if type(hash) == Block: + hash = hash.getHash() + + # just to make sure someone didn't put in a bool or something lol + hash = str(hash) + + # if it exists, return its content + if hash in Block.getCache(): + return Block.blockCache[hash] + + return None + + def cache(block, override = False): + # why even bother if they're giving bad data? + if not type(block) == Block: + return False + + # only cache if written to file + if block.getHash() is None: + return False + + # if it's already cached, what are we here for? + if block.getHash() in Block.getCache() and not override: + return False + + # dump old cached blocks if the size exeeds the maximum + if sys.getsizeof(Block.blockCacheOrder) >= config.get('allocations.blockCacheTotal', 50000000): # 50MB default cache size + del Block.blockCache[blockCacheOrder.pop(0)] + + # cache block content + Block.blockCache[block.getHash()] = block.getRaw() + Block.blockCacheOrder.append(block.getHash()) + + return True diff --git a/onionr/onionrexceptions.py b/onionr/onionrexceptions.py index 3f201b80..43077e67 100644 --- a/onionr/onionrexceptions.py +++ b/onionr/onionrexceptions.py @@ -18,7 +18,16 @@ along with this program. If not, see . ''' +# general exceptions +class NotFound(Exception): + pass +class Unknown(Exception): + pass +class Invalid(Exception): + pass + +# network level exceptions class MissingPort(Exception): pass class InvalidAddress(Exception): - pass \ No newline at end of file + pass diff --git a/onionr/onionrplugins.py b/onionr/onionrplugins.py index c6f94a3b..6160838e 100644 --- a/onionr/onionrplugins.py +++ b/onionr/onionrplugins.py @@ -64,9 +64,7 @@ def enable(name, onionr = None, start_event = True): enabled_plugins = get_enabled_plugins() if not name in enabled_plugins: enabled_plugins.append(name) - config_plugins = config.get('plugins') - config_plugins['enabled'] = enabled_plugins - config.set('plugins', config_plugins, True) + config.set('plugins.enabled', enabled_plugins, True) events.call(get_plugin(name), 'enable', onionr) @@ -93,9 +91,7 @@ def disable(name, onionr = None, stop_event = True): if is_enabled(name): enabled_plugins = get_enabled_plugins() enabled_plugins.remove(name) - config_plugins = config.get('plugins') - config_plugins['enabled'] = enabled_plugins - config.set('plugins', config_plugins, True) + config.set('plugins.enabled', enabled_plugins, True) if exists(name): events.call(get_plugin(name), 'disable', onionr) @@ -187,7 +183,7 @@ def get_enabled_plugins(): config.reload() - return config.get('plugins')['enabled'] + return config.get('plugins.enabled', list()) def is_enabled(name): ''' diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index de32e415..2d05acf6 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -177,7 +177,7 @@ class OnionrUtils: self.getTimeBypassToken() # TODO: URL encode parameters, just as an extra measure. May not be needed, but should be added regardless. try: - retData = requests.get('http://' + open('data/host.txt', 'r').read() + ':' + str(config.get('client')['port']) + '/client/?action=' + command + '&token=' + str(config.get('client')['client_hmac']) + '&timingToken=' + self.timingToken).text + retData = requests.get('http://%s:%s/client/?action=%s&token=%s&timingToken=' % (open('data/host.txt', 'r').read(), config.get('client.port', 59496), command, config.get('client.hmac'), self.timingToken)).text except Exception as error: if not silent: logger.error('Failed to make local request (command: %s).' % command, error=error) diff --git a/onionr/static-data/default_config.json b/onionr/static-data/default_config.json index a4fc7c9d..40958e97 100644 --- a/onionr/static-data/default_config.json +++ b/onionr/static-data/default_config.json @@ -1,8 +1,18 @@ { - "devmode": true, - "dc_response": true, + "general" : { + "dev_mode": true, + "display_header" : true, + + "newCommunicator": false, + + "dc_response": true, + "dc_execcallbacks" : true + }, + + "client" : { + + }, - "display_header" : true, "log": { "file": { "output": true, @@ -15,14 +25,20 @@ } }, + "tor" : { + + }, + "i2p":{ - "host": false, - "connect": true, - "ownAddr": "" + "host": false, + "connect": true, + "ownAddr": "" }, + "allocations":{ "disk": 1000000000, - "netTotal": 1000000000 - }, - "newCommunicator": false + "netTotal": 1000000000, + "blockCache" : 5000000, + "blockCacheTotal" : 50000000 + } }