Refactor Onionr

This commit is contained in:
Arinerron 2018-06-13 21:17:58 -07:00
parent 6cb69c7187
commit af237eab0b
11 changed files with 184 additions and 67 deletions

View File

@ -52,7 +52,7 @@ class API:
config.reload() config.reload()
if config.get('devmode', True): if config.get('dev_mode', True):
self._developmentMode = True self._developmentMode = True
logger.set_level(logger.LEVEL_DEBUG) logger.set_level(logger.LEVEL_DEBUG)
else: else:
@ -65,12 +65,12 @@ class API:
self._crypto = onionrcrypto.OnionrCrypto(self._core) self._crypto = onionrcrypto.OnionrCrypto(self._core)
self._utils = onionrutils.OnionrUtils(self._core) self._utils = onionrutils.OnionrUtils(self._core)
app = flask.Flask(__name__) app = flask.Flask(__name__)
bindPort = int(config.get('client')['port']) bindPort = int(config.get('client.port', 59496))
self.bindPort = bindPort self.bindPort = bindPort
self.clientToken = config.get('client')['client_hmac'] self.clientToken = config.get('client.hmac')
self.timeBypassToken = base64.b16encode(os.urandom(32)).decode() self.timeBypassToken = base64.b16encode(os.urandom(32)).decode()
self.i2pEnabled = config.get('i2p', {'host' : False})['host'] self.i2pEnabled = config.get('i2p.host', False)
self.mimeType = 'text/plain' self.mimeType = 'text/plain'

View File

@ -75,7 +75,7 @@ class OnionrCommunicate:
plugins.reload() plugins.reload()
# Print nice header thing :) # Print nice header thing :)
if config.get('display_header', True): if config.get('general.display_header', True):
self.header() self.header()
while True: while True:
@ -149,7 +149,7 @@ class OnionrCommunicate:
logger.info('Checking for callbacks with connection %s...' % data['id']) logger.info('Checking for callbacks with connection %s...' % data['id'])
self.check_callbacks(data, config.get('dc_execcallbacks', True)) self.check_callbacks(data, config.get('general.dc_execcallbacks', True))
events.event('incoming_direct_connection', data = {'callback' : True, 'communicator' : self, 'data' : data}) events.event('incoming_direct_connection', data = {'callback' : True, 'communicator' : self, 'data' : data})
except Exception as e: except Exception as e:
@ -349,7 +349,7 @@ class OnionrCommunicate:
If yet another callback is requested, it can be put in the `callback` parameter. If yet another callback is requested, it can be put in the `callback` parameter.
''' '''
if config.get('dc_response', True): if config.get('general.dc_response', True):
data['id'] = identifier data['id'] = identifier
data['sender'] = open('data/hs/hostname').read() data['sender'] = open('data/hs/hostname').read()
data['callback'] = True data['callback'] = True
@ -768,7 +768,7 @@ class OnionrCommunicate:
shouldRun = False shouldRun = False
debug = True debug = True
developmentMode = False developmentMode = False
if config.get('devmode', True): if config.get('general.dev_mode', True):
developmentMode = True developmentMode = True
try: try:
if sys.argv[1] == 'run': if sys.argv[1] == 'run':

View File

@ -46,7 +46,7 @@ class OnionrCommunicatorDaemon:
plugins.reload() plugins.reload()
# Print nice header thing :) # Print nice header thing :)
if config.get('display_header', True): if config.get('general.display_header', True):
self.header() self.header()
if debug or developmentMode: if debug or developmentMode:
@ -215,7 +215,7 @@ class OnionrCommunicatorTimers:
shouldRun = False shouldRun = False
debug = True debug = True
developmentMode = False developmentMode = False
if config.get('devmode', True): if config.get('general.dev_mode', True):
developmentMode = True developmentMode = True
try: try:
if sys.argv[1] == 'run': if sys.argv[1] == 'run':

View File

@ -28,9 +28,20 @@ def get(key, default = None):
Gets the key from configuration, or returns `default` Gets the key from configuration, or returns `default`
''' '''
if is_set(key): key = str(key).split('.')
return get_config()[key] data = _config
return default
last = key.pop()
for item in key:
if (not item in data) or (not type(data[item]) == dict):
return default
data = data[item]
if not last in data:
return default
return data[last]
def set(key, value = None, savefile = False): def set(key, value = None, savefile = False):
''' '''
@ -38,16 +49,40 @@ def set(key, value = None, savefile = False):
''' '''
global _config global _config
key = str(key).split('.')
data = _config
last = key.pop()
for item in key:
if (not item in data) or (not type(data[item]) == dict):
data[item] = dict()
data = data[item]
if value is None: if value is None:
del _config[key] del data[last]
else: else:
_config[key] = value data[last] = value
if savefile: if savefile:
save() save()
def is_set(key): def is_set(key):
return key in get_config() and not get_config()[key] is None key = str(key).split('.')
data = _config
last = key.pop()
for item in key:
if (not item in data) or (not type(data[item]) == dict):
return False
data = data[item]
if not last in data:
return False
return True
def check(): def check():
''' '''

View File

@ -110,7 +110,7 @@ class Core:
''' '''
Add an address to the address database (only tor currently) Add an address to the address database (only tor currently)
''' '''
if address == config.get('i2p', {'ownAddr' : None})['ownAddr']: if address == config.get('i2p.ownAddr', None):
return False return False
if self._utils.validateID(address): if self._utils.validateID(address):

View File

@ -67,22 +67,22 @@ class Onionr:
config.set_config(json.loads(open('static-data/default_config.json').read())) # this is the default config, it will be overwritten if a config file already exists. Else, it saves it config.set_config(json.loads(open('static-data/default_config.json').read())) # this is the default config, it will be overwritten if a config file already exists. Else, it saves it
else: else:
# the default config file doesn't exist, try hardcoded config # the default config file doesn't exist, try hardcoded config
config.set_config({'devmode': True, 'log': {'file': {'output': True, 'path': 'data/output.log'}, 'console': {'output': True, 'color': True}}}) config.set_config({'dev_mode': True, 'log': {'file': {'output': True, 'path': 'data/output.log'}, 'console': {'output': True, 'color': True}}})
if not data_exists: if not data_exists:
config.save() config.save()
config.reload() # this will read the configuration file into memory config.reload() # this will read the configuration file into memory
settings = 0b000 settings = 0b000
if config.get('log', {'console': {'color': True}})['console']['color']: if config.get('log.console.color', True):
settings = settings | logger.USE_ANSI settings = settings | logger.USE_ANSI
if config.get('log', {'console': {'output': True}})['console']['output']: if config.get('log.console.output', True):
settings = settings | logger.OUTPUT_TO_CONSOLE settings = settings | logger.OUTPUT_TO_CONSOLE
if config.get('log', {'file': {'output': True}})['file']['output']: if config.get('log.file.output', True):
settings = settings | logger.OUTPUT_TO_FILE settings = settings | logger.OUTPUT_TO_FILE
logger.set_file(config.get('log', {'file': {'path': 'data/output.log'}})['file']['path']) logger.set_file(config.get('log.file.path', '/tmp/onionr.log'))
logger.set_settings(settings) logger.set_settings(settings)
if str(config.get('devmode', True)).lower() == 'true': if str(config.get('general.dev_mode', True)).lower() == 'true':
self._developmentMode = True self._developmentMode = True
logger.set_level(logger.LEVEL_DEBUG) logger.set_level(logger.LEVEL_DEBUG)
else: else:
@ -147,7 +147,7 @@ class Onionr:
randomPort = random.randint(1024, 65535) randomPort = random.randint(1024, 65535)
if self.onionrUtils.checkPort(randomPort): if self.onionrUtils.checkPort(randomPort):
break break
config.set('client', {'participate': 'true', 'client_hmac': base64.b16encode(os.urandom(32)).decode('utf-8'), 'port': randomPort, 'api_version': API_VERSION}, True) config.set('client', {'participate': 'true', 'hmac': base64.b16encode(os.urandom(32)).decode('utf-8'), 'port': randomPort, 'api_version': API_VERSION}, True)
self.cmds = { self.cmds = {
'': self.showHelpSuggestion, '': self.showHelpSuggestion,
@ -260,7 +260,7 @@ class Onionr:
self.onionrCore.daemonQueueAdd('connectedPeers') self.onionrCore.daemonQueueAdd('connectedPeers')
def getWebPassword(self): def getWebPassword(self):
return config.get('client')['client_hmac'] return config.get('client.hmac')
def getHelp(self): def getHelp(self):
return self.cmdhelp return self.cmdhelp
@ -532,12 +532,12 @@ class Onionr:
logger.info('Do ' + logger.colors.bold + sys.argv[0] + ' --help' + logger.colors.reset + logger.colors.fg.green + ' for Onionr help.') logger.info('Do ' + logger.colors.bold + sys.argv[0] + ' --help' + logger.colors.reset + logger.colors.fg.green + ' for Onionr help.')
def start(self, input = False): def start(self, input = False, override = False):
''' '''
Starts the Onionr daemon Starts the Onionr daemon
''' '''
if os.path.exists('.onionr-lock'): if os.path.exists('.onionr-lock') and not override:
logger.fatal('Cannot start. Daemon is already running, or it did not exit cleanly.\n(if you are sure that there is not a daemon running, delete .onionr-lock & try again).') logger.fatal('Cannot start. Daemon is already running, or it did not exit cleanly.\n(if you are sure that there is not a daemon running, delete .onionr-lock & try again).')
else: else:
if not self.debug and not self._developmentMode: if not self.debug and not self._developmentMode:
@ -558,7 +558,7 @@ class Onionr:
if not os.environ.get("WERKZEUG_RUN_MAIN") == "true": if not os.environ.get("WERKZEUG_RUN_MAIN") == "true":
if self._developmentMode: if self._developmentMode:
logger.warn('DEVELOPMENT MODE ENABLED (THIS IS LESS SECURE!)', timestamp = False) logger.warn('DEVELOPMENT MODE ENABLED (THIS IS LESS SECURE!)', timestamp = False)
net = NetController(config.get('client')['port']) net = NetController(config.get('client.port', 59496))
logger.info('Tor is starting...') logger.info('Tor is starting...')
if not net.startTor(): if not net.startTor():
sys.exit(1) sys.exit(1)
@ -566,7 +566,7 @@ class Onionr:
logger.info('Our Public key: ' + self.onionrCore._crypto.pubKey) logger.info('Our Public key: ' + self.onionrCore._crypto.pubKey)
time.sleep(1) time.sleep(1)
try: try:
if config.get('newCommunicator'): if config.get('general.newCommunicator', False):
communicatorDaemon = './communicator2.py' communicatorDaemon = './communicator2.py'
logger.info('Using new communicator') logger.info('Using new communicator')
except NameError: except NameError:
@ -586,7 +586,7 @@ class Onionr:
logger.warn('Killing the running daemon...', timestamp = False) logger.warn('Killing the running daemon...', timestamp = False)
try: try:
events.event('daemon_stop', onionr = self) events.event('daemon_stop', onionr = self)
net = NetController(config.get('client')['port']) net = NetController(config.get('client.port', 59496))
try: try:
self.onionrUtils.localCommand('shutdown') self.onionrUtils.localCommand('shutdown')
except requests.exceptions.ConnectionError: except requests.exceptions.ConnectionError:
@ -625,7 +625,7 @@ class Onionr:
# count stats # count stats
'div2' : True, 'div2' : True,
'Known Peers Count' : str(len(self.onionrCore.listPeers()) - 1), 'Known Peers Count' : str(len(self.onionrCore.listPeers()) - 1),
'Enabled Plugins Count' : str(len(config.get('plugins')['enabled'])) + ' / ' + str(len(os.listdir('data/plugins/'))), 'Enabled Plugins Count' : str(len(config.get('plugins.enabled', list()))) + ' / ' + str(len(os.listdir('data/plugins/'))),
'Known Blocks Count' : str(totalBlocks), 'Known Blocks Count' : str(totalBlocks),
'Percent Blocks Signed' : str(round(100 * signedBlocks / max(totalBlocks, 1), 2)) + '%' 'Percent Blocks Signed' : str(round(100 * signedBlocks / max(totalBlocks, 1), 2)) + '%'
} }

View File

@ -18,10 +18,13 @@
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
import core as onionrcore, logger import core as onionrcore, logger, config
import json, os, datetime, base64 import json, os, sys, datetime, base64
class Block: class Block:
blockCacheOrder = list() # NEVER write your own code that writes to this!
blockCache = dict() # should never be accessed directly, look at Block.getCache()
def __init__(self, hash = None, core = None): def __init__(self, hash = None, core = None):
''' '''
Initializes Onionr Initializes Onionr
@ -43,6 +46,9 @@ class Block:
self.hash = None self.hash = None
self.core = None self.core = None
else: else:
if type(hash) == bytes:
hash = hash.decode()
self.btype = '' self.btype = ''
self.bcontent = '' self.bcontent = ''
self.hash = hash self.hash = hash
@ -65,7 +71,8 @@ class Block:
if self.getCore() is None: if self.getCore() is None:
self.core = onionrcore.Core() self.core = onionrcore.Core()
if not self.getHash() is None: if not self.getHash() is None:
self.update() if not self.update():
logger.debug('Failed to open block %s.' % self.getHash())
# logic # logic
@ -93,16 +100,23 @@ class Block:
if blockdata is None: if blockdata is None:
filelocation = file filelocation = file
readfile = True
if filelocation is None: if filelocation is None:
if self.getHash() is None: if self.getHash() is None:
return False return False
try: elif self.getHash() in Block.getCache():
filelocation = 'data/blocks/%s.dat' % self.getHash().decode() # get the block from cache, if it's in it
except AttributeError: blockdata = Block.getCache(self.getHash())
readfile = False
# read from file if it's still None
if blockdata is None:
filelocation = 'data/blocks/%s.dat' % self.getHash() filelocation = 'data/blocks/%s.dat' % self.getHash()
with open(filelocation, 'rb') as f: if readfile:
blockdata = f.read().decode('utf-8') with open(filelocation, 'rb') as f:
blockdata = f.read().decode()
self.blockFile = filelocation self.blockFile = filelocation
else: else:
@ -126,6 +140,10 @@ class Block:
self.date = datetime.datetime.fromtimestamp(self.getDate()) self.date = datetime.datetime.fromtimestamp(self.getDate())
self.valid = True self.valid = True
if len(self.getRaw()) <= config.get('allocations.blockCache', 500000):
self.cache()
return True return True
except Exception as e: except Exception as e:
logger.error('Failed to update block data.', error = e, timestamp = False) logger.error('Failed to update block data.', error = e, timestamp = False)
@ -641,11 +659,54 @@ class Block:
- (bool): whether or not the block file exists - (bool): whether or not the block file exists
''' '''
# no input data? scrap it.
if hash is None: if hash is None:
return False return False
elif type(hash) == Block:
if type(hash) == Block:
blockfile = hash.getBlockFile() blockfile = hash.getBlockFile()
else: else:
blockfile = 'data/blocks/%s.dat' % hash blockfile = 'data/blocks/%s.dat' % hash
return os.path.exists(blockfile) and os.path.isfile(blockfile) return os.path.exists(blockfile) and os.path.isfile(blockfile)
def getCache(hash = None):
# give a list of the hashes of the cached blocks
if hash is None:
return list(Block.blockCache.keys())
# if they inputted self or a Block, convert to hash
if type(hash) == Block:
hash = hash.getHash()
# just to make sure someone didn't put in a bool or something lol
hash = str(hash)
# if it exists, return its content
if hash in Block.getCache():
return Block.blockCache[hash]
return None
def cache(block, override = False):
# why even bother if they're giving bad data?
if not type(block) == Block:
return False
# only cache if written to file
if block.getHash() is None:
return False
# if it's already cached, what are we here for?
if block.getHash() in Block.getCache() and not override:
return False
# dump old cached blocks if the size exeeds the maximum
if sys.getsizeof(Block.blockCacheOrder) >= config.get('allocations.blockCacheTotal', 50000000): # 50MB default cache size
del Block.blockCache[blockCacheOrder.pop(0)]
# cache block content
Block.blockCache[block.getHash()] = block.getRaw()
Block.blockCacheOrder.append(block.getHash())
return True

View File

@ -18,6 +18,15 @@
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
# general exceptions
class NotFound(Exception):
pass
class Unknown(Exception):
pass
class Invalid(Exception):
pass
# network level exceptions
class MissingPort(Exception): class MissingPort(Exception):
pass pass
class InvalidAddress(Exception): class InvalidAddress(Exception):

View File

@ -64,9 +64,7 @@ def enable(name, onionr = None, start_event = True):
enabled_plugins = get_enabled_plugins() enabled_plugins = get_enabled_plugins()
if not name in enabled_plugins: if not name in enabled_plugins:
enabled_plugins.append(name) enabled_plugins.append(name)
config_plugins = config.get('plugins') config.set('plugins.enabled', enabled_plugins, True)
config_plugins['enabled'] = enabled_plugins
config.set('plugins', config_plugins, True)
events.call(get_plugin(name), 'enable', onionr) events.call(get_plugin(name), 'enable', onionr)
@ -93,9 +91,7 @@ def disable(name, onionr = None, stop_event = True):
if is_enabled(name): if is_enabled(name):
enabled_plugins = get_enabled_plugins() enabled_plugins = get_enabled_plugins()
enabled_plugins.remove(name) enabled_plugins.remove(name)
config_plugins = config.get('plugins') config.set('plugins.enabled', enabled_plugins, True)
config_plugins['enabled'] = enabled_plugins
config.set('plugins', config_plugins, True)
if exists(name): if exists(name):
events.call(get_plugin(name), 'disable', onionr) events.call(get_plugin(name), 'disable', onionr)
@ -187,7 +183,7 @@ def get_enabled_plugins():
config.reload() config.reload()
return config.get('plugins')['enabled'] return config.get('plugins.enabled', list())
def is_enabled(name): def is_enabled(name):
''' '''

View File

@ -177,7 +177,7 @@ class OnionrUtils:
self.getTimeBypassToken() self.getTimeBypassToken()
# TODO: URL encode parameters, just as an extra measure. May not be needed, but should be added regardless. # TODO: URL encode parameters, just as an extra measure. May not be needed, but should be added regardless.
try: try:
retData = requests.get('http://' + open('data/host.txt', 'r').read() + ':' + str(config.get('client')['port']) + '/client/?action=' + command + '&token=' + str(config.get('client')['client_hmac']) + '&timingToken=' + self.timingToken).text retData = requests.get('http://%s:%s/client/?action=%s&token=%s&timingToken=' % (open('data/host.txt', 'r').read(), config.get('client.port', 59496), command, config.get('client.hmac'), self.timingToken)).text
except Exception as error: except Exception as error:
if not silent: if not silent:
logger.error('Failed to make local request (command: %s).' % command, error=error) logger.error('Failed to make local request (command: %s).' % command, error=error)

View File

@ -1,8 +1,18 @@
{ {
"devmode": true, "general" : {
"dc_response": true, "dev_mode": true,
"display_header" : true,
"newCommunicator": false,
"dc_response": true,
"dc_execcallbacks" : true
},
"client" : {
},
"display_header" : true,
"log": { "log": {
"file": { "file": {
"output": true, "output": true,
@ -15,14 +25,20 @@
} }
}, },
"tor" : {
},
"i2p":{ "i2p":{
"host": false, "host": false,
"connect": true, "connect": true,
"ownAddr": "" "ownAddr": ""
}, },
"allocations":{ "allocations":{
"disk": 1000000000, "disk": 1000000000,
"netTotal": 1000000000 "netTotal": 1000000000,
}, "blockCache" : 5000000,
"newCommunicator": false "blockCacheTotal" : 50000000
}
} }