diff --git a/onionr/api.py b/onionr/api.py index 1416a1ea..48587f8e 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -70,7 +70,7 @@ class API: self.clientToken = config.get('client')['client_hmac'] self.timeBypassToken = base64.b16encode(os.urandom(32)).decode() - self.i2pEnabled = config.get('i2p')['host'] + self.i2pEnabled = config.get('i2p', {'host' : False})['host'] self.mimeType = 'text/plain' @@ -85,9 +85,9 @@ class API: self.host = '127.' + str(hostNums[0]) + '.' + str(hostNums[1]) + '.' + str(hostNums[2]) else: self.host = '127.0.0.1' - hostFile = open('data/host.txt', 'w') - hostFile.write(self.host) - hostFile.close() + + with open('data/host.txt', 'w') as file: + file.write(self.host) @app.before_request def beforeReq(): @@ -259,7 +259,7 @@ class API: return resp if not os.environ.get("WERKZEUG_RUN_MAIN") == "true": - logger.info('Starting client on ' + self.host + ':' + str(bindPort) + '...', timestamp=True) + logger.info('Starting client on ' + self.host + ':' + str(bindPort) + '...', timestamp=False) try: self.http_server = WSGIServer((self.host, bindPort), app) diff --git a/onionr/communicator.py b/onionr/communicator.py index 3661b6bb..48e40bf9 100755 --- a/onionr/communicator.py +++ b/onionr/communicator.py @@ -123,16 +123,16 @@ class OnionrCommunicate: announceAttempts = 3 announceAttemptCount = 0 announceVal = False - logger.info('Announcing node to ' + command[1], timestamp=True) + logger.info('Announcing node to %s...' % command[1], timestamp=True) while not announceVal: announceAttemptCount += 1 announceVal = self.performGet('announce', command[1], data=self._core.hsAdder.replace('\n', ''), skipHighFailureAddress=True) - logger.info(announceVal) + # logger.info(announceVal) if announceAttemptCount >= announceAttempts: - logger.warn('Unable to announce to ' + command[1]) + logger.warn('Unable to announce to %s' % command[1]) break elif command[0] == 'runCheck': - logger.info('Status check; looks good.') + logger.debug('Status check; looks good.') open('data/.runcheck', 'w+').close() elif command[0] == 'kex': self.pexCount = pexTimer - 1 @@ -188,13 +188,17 @@ class OnionrCommunicate: id_peer_cache = {} def registerTimer(self, timerName, rate, timerFunc=None): - '''Register a communicator timer''' + ''' + Register a communicator timer + ''' self.communicatorTimers[timerName] = rate self.communicatorTimerCounts[timerName] = 0 self.communicatorTimerFuncs[timerName] = timerFunc def timerTick(self): - '''Increments timers "ticks" and calls funcs if applicable''' + ''' + Increments timers "ticks" and calls funcs if applicable + ''' tName = '' for i in self.communicatorTimers.items(): tName = i[0] @@ -617,7 +621,9 @@ class OnionrCommunicate: return def removeBlockFromProcessingList(self, block): - '''Remove a block from the processing list''' + ''' + Remove a block from the processing list + ''' try: self.blocksProcessing.remove(block) except ValueError: @@ -724,7 +730,8 @@ class OnionrCommunicate: r = requests.get(url, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30)) retData = r.text except requests.exceptions.RequestException as e: - logger.debug("%s failed with peer %s" % (action, peer)) + logger.debug('%s failed with peer %s' % (action, peer)) + logger.debug('Error: %s' % str(e)) retData = False if not retData: diff --git a/onionr/netcontroller.py b/onionr/netcontroller.py index a40dab86..7ad74cc0 100644 --- a/onionr/netcontroller.py +++ b/onionr/netcontroller.py @@ -103,7 +103,7 @@ DataDirectory data/tordata/ logger.fatal("Got keyboard interrupt") return False - logger.info('Finished starting Tor', timestamp=True) + logger.debug('Finished starting Tor.', timestamp=True) self.readyState = True myID = open('data/hs/hostname', 'r') diff --git a/onionr/onionr.py b/onionr/onionr.py index d4d7e3ea..4bff79f9 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -31,6 +31,7 @@ import api, core, config, logger, onionrplugins as plugins, onionrevents as even import onionrutils from onionrutils import OnionrUtils from netcontroller import NetController +from onionrblockapi import Block try: from urllib3.contrib.socks import SOCKSProxyManager @@ -192,8 +193,11 @@ class Onionr: 'add-addr': self.addAddress, 'addaddr': self.addAddress, 'addaddress': self.addAddress, + + 'add-file': self.addFile, 'addfile': self.addFile, + 'import-blocks': self.onionrUtils.importNewBlocks, 'importblocks': self.onionrUtils.importNewBlocks, 'introduce': self.onionrCore.introduceNode, @@ -216,8 +220,8 @@ class Onionr: 'add-msg': 'Broadcasts a message to the Onionr network', 'pm': 'Adds a private message to block', 'get-pms': 'Shows private messages sent to you', - 'addfile': 'Create an Onionr block from a file', - 'importblocks': 'import blocks from the disk (Onionr is transport-agnostic!)', + 'add-file': 'Create an Onionr block from a file', + 'import-blocks': 'import blocks from the disk (Onionr is transport-agnostic!)', 'introduce': 'Introduce your node to the public Onionr network', } @@ -391,12 +395,11 @@ class Onionr: except KeyboardInterrupt: return - #addedHash = self.onionrCore.setData(messageToAdd) - addedHash = self.onionrCore.insertBlock(messageToAdd, header='txt') - #self.onionrCore.addToBlockDB(addedHash, selfInsert=True) - #self.onionrCore.setBlockType(addedHash, 'txt') - if addedHash != '': + addedHash = Block('txt', messageToAdd).save() + if addedHash != None: logger.info("Message inserted as as block %s" % addedHash) + else: + logger.error('Failed to insert block.', timestamp = False) return def getPMs(self): @@ -520,12 +523,12 @@ class Onionr: if not os.environ.get("WERKZEUG_RUN_MAIN") == "true": if self._developmentMode: - logger.warn('DEVELOPMENT MODE ENABLED (THIS IS LESS SECURE!)') + logger.warn('DEVELOPMENT MODE ENABLED (THIS IS LESS SECURE!)', timestamp = False) net = NetController(config.get('client')['port']) logger.info('Tor is starting...') if not net.startTor(): sys.exit(1) - logger.info('Started Tor .onion service: ' + logger.colors.underline + net.myID) + logger.info('Started .onion service: ' + logger.colors.underline + net.myID) logger.info('Our Public key: ' + self.onionrCore._crypto.pubKey) time.sleep(1) subprocess.Popen(["./communicator.py", "run", str(net.socksPort)]) @@ -562,6 +565,9 @@ class Onionr: try: # define stats messages here + totalBlocks = len(Block.getBlocks()) + signedBlocks = len(Block.getBlocks(signed = True)) + messages = { # info about local client 'Onionr Daemon Status' : ((logger.colors.fg.green + 'Online') if self.onionrUtils.isCommunicatorRunning(timeout = 2) else logger.colors.fg.red + 'Offline'), @@ -577,7 +583,9 @@ class Onionr: # count stats 'div2' : True, 'Known Peers Count' : str(len(self.onionrCore.listPeers()) - 1), - 'Enabled Plugins Count' : str(len(config.get('plugins')['enabled'])) + ' / ' + str(len(os.listdir('data/plugins/'))) + 'Enabled Plugins Count' : str(len(config.get('plugins')['enabled'])) + ' / ' + str(len(os.listdir('data/plugins/'))), + 'Known Blocks Count' : str(totalBlocks), + 'Percent Blocks Signed' : str(round(100 * signedBlocks / totalBlocks, 2)) + '%' } # color configuration @@ -639,18 +647,30 @@ class Onionr: return None def addFile(self): - '''command to add a file to the onionr network''' - if len(sys.argv) >= 2: - newFile = sys.argv[2] - logger.info('Attempting to add file...') - try: - with open(newFile, 'rb') as new: - new = new.read() - except FileNotFoundError: + ''' + Adds a file to the onionr network + ''' + + if len(sys.argv) >= 3: + filename = sys.argv[2] + contents = None + + if not os.path.exists(filename): logger.warn('That file does not exist. Improper path?') + + try: + with open(filename, 'rb') as file: + contents = file.read().decode() + except: + pass + + if not contents is None: + blockhash = Block('bin', contents).save() + logger.info('File %s saved in block %s.' % (filename, blockhash)) else: - logger.debug(new) - logger.info(self.onionrCore.insertBlock(new, header='bin')) + logger.error('Failed to save file in block.', timestamp = False) + else: + logger.error('%s add-file ' % sys.argv[0], timestamp = False) Onionr() diff --git a/onionr/onionrplugins.py b/onionr/onionrplugins.py index 175b2336..c6f94a3b 100644 --- a/onionr/onionrplugins.py +++ b/onionr/onionrplugins.py @@ -77,7 +77,7 @@ def enable(name, onionr = None, start_event = True): else: return False else: - logger.error('Failed to enable plugin \"' + name + '\", disabling plugin.') + logger.error('Failed to enable plugin \"%s\", disabling plugin.' % name) disable(name) return False @@ -121,9 +121,9 @@ def start(name, onionr = None): return plugin except: - logger.error('Failed to start module \"' + name + '\".') + logger.error('Failed to start module \"%s\".' % name) else: - logger.error('Failed to start nonexistant module \"' + name + '\".') + logger.error('Failed to start nonexistant module \"%s\".' % name) return None @@ -145,9 +145,9 @@ def stop(name, onionr = None): return plugin except: - logger.error('Failed to stop module \"' + name + '\".') + logger.error('Failed to stop module \"%s\".' % name) else: - logger.error('Failed to stop nonexistant module \"' + name + '\".') + logger.error('Failed to stop nonexistant module \"%s\".' % name) return None diff --git a/onionr/onionrproofs.py b/onionr/onionrproofs.py index bf749ad6..c1b3fb41 100644 --- a/onionr/onionrproofs.py +++ b/onionr/onionrproofs.py @@ -22,7 +22,7 @@ import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, lo import core class POW: - def pow(self, reporting = False): + def pow(self, reporting = False, myCore = None): startTime = math.floor(time.time()) self.hashing = True self.reporting = reporting @@ -30,7 +30,7 @@ class POW: answer = '' heartbeat = 200000 hbCount = 0 - myCore = core.Core() + while self.hashing: rand = nacl.utils.random() token = nacl.hash.blake2b(rand + self.data).decode() @@ -39,23 +39,22 @@ class POW: self.hashing = False iFound = True break - else: - logger.debug('POW thread exiting, another thread found result') + if iFound: endTime = math.floor(time.time()) if self.reporting: - logger.info('Found token ' + token, timestamp=True) - logger.info('rand value: ' + base64.b64encode(rand).decode()) - logger.info('took ' + str(endTime - startTime) + ' seconds', timestamp=True) + logger.debug('Found token after %s seconds: %s' % (endTime - startTime, token), timestamp=True) + logger.debug('Random value was: %s' % base64.b64encode(rand).decode()) self.result = (token, rand) - def __init__(self, data): + def __init__(self, data, threadCount = 5): self.foundHash = False self.difficulty = 0 self.data = data + self.threadCount = threadCount dataLen = sys.getsizeof(data) - self.difficulty = math.floor(dataLen/1000000) + self.difficulty = math.floor(dataLen / 1000000) if self.difficulty <= 2: self.difficulty = 4 @@ -63,19 +62,19 @@ class POW: self.data = self.data.encode() except AttributeError: pass + self.data = nacl.hash.blake2b(self.data) - logger.debug('Computing difficulty of ' + str(self.difficulty)) + logger.info('Computing POW (difficulty: %s)...' % self.difficulty) - self.mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode() - self.puzzle = self.mainHash[0:self.difficulty] - #logger.debug('trying to find ' + str(self.mainHash)) - tOne = threading.Thread(name='one', target=self.pow, args=(True,)) - tTwo = threading.Thread(name='two', target=self.pow, args=(True,)) - tThree = threading.Thread(name='three', target=self.pow, args=(True,)) - tOne.start() - tTwo.start() - tThree.start() + self.mainHash = '0' * 70 + self.puzzle = self.mainHash[0:min(self.difficulty, len(self.mainHash))] + + myCore = core.Core() + for i in range(max(1, threadCount)): + t = threading.Thread(name = 'thread%s' % i, target = self.pow, args = (True,myCore)) + t.start() + return def shutdown(self): @@ -89,9 +88,11 @@ class POW: ''' Returns the result then sets to false, useful to automatically clear the result ''' + try: retVal = self.result except AttributeError: retVal = False + self.result = False return retVal diff --git a/onionr/static-data/default-plugins/pluginmanager/main.py b/onionr/static-data/default-plugins/pluginmanager/main.py index 59c23d44..4812cc90 100644 --- a/onionr/static-data/default-plugins/pluginmanager/main.py +++ b/onionr/static-data/default-plugins/pluginmanager/main.py @@ -34,9 +34,9 @@ def writeKeys(): Serializes and writes the keystore in memory to file ''' - file = open(keys_file, 'w') - file.write(json.dumps(keys_data, indent=4, sort_keys=True)) - file.close() + with open(keys_file, 'w') as file: + file.write(json.dumps(keys_data, indent=4, sort_keys=True)) + file.close() def readKeys(): ''' @@ -44,7 +44,8 @@ def readKeys(): ''' global keys_data - keys_data = json.loads(open(keys_file).read()) + with open(keys_file) as file: + keys_data = json.loads(file.read()) return keys_data def getKey(plugin): @@ -106,27 +107,37 @@ def getRepositories(): readKeys() return keys_data['repositories'] -def addRepository(repositories, data): +def addRepository(blockhash, data): ''' Saves the plugin name, to remember that it was installed by the pluginmanager ''' global keys_data readKeys() - keys_data['repositories'][repositories] = data + keys_data['repositories'][blockhash] = data writeKeys() -def removeRepository(repositories): +def removeRepository(blockhash): ''' Removes the plugin name from the pluginmanager's records ''' global keys_data readKeys() - if plugin in keys_data['repositories']: - del keys_data['repositories'][repositories] + if blockhash in keys_data['repositories']: + del keys_data['repositories'][blockhash] writeKeys() +def createRepository(plugins): + contents = {'plugins' : plugins, 'author' : getpass.getuser(), 'compiled-by' : plugin_name} + + block = Block(core = pluginapi.get_core()) + + block.setType('repository') + block.setContent(json.dumps(contents)) + + return block.save(True) + def check(): ''' Checks to make sure the keystore file still exists @@ -144,7 +155,7 @@ def sanitize(name): def blockToPlugin(block): try: - block = Block(block) + block = Block(block, core = pluginapi.get_core()) blockContent = json.loads(block.getContent()) name = sanitize(blockContent['name']) @@ -194,14 +205,19 @@ def pluginToBlock(plugin, import_block = True): shutil.rmtree(directory + '__pycache__') shutil.make_archive(zipfile[:-4], 'zip', directory) - data = base64.b64encode(open(zipfile, 'rb').read()) + data = '' + with open(zipfile, 'rb') as file: + data = base64.b64encode(file.read()) author = getpass.getuser() description = 'Default plugin description' info = {"name" : plugin} try: if os.path.exists(directory + 'info.json'): - info = json.loads(open(directory + 'info.json').read()) + info = '' + with open(directory + 'info.json').read() as file: + info = json.loads(file.read()) + if 'author' in info: author = info['author'] if 'description' in info: @@ -211,7 +227,13 @@ def pluginToBlock(plugin, import_block = True): metadata = {'author' : author, 'date' : str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')), 'name' : plugin, 'info' : info, 'compiled-by' : plugin_name, 'content' : data.decode('utf-8'), 'description' : description} - hash = pluginapi.get_core().insertBlock(json.dumps(metadata), header = 'plugin', sign = True) + block = Block(core = pluginapi.get_core()) + + block.setType('plugin') + block.setContent(json.dumps(metadata)) + + hash = block.save(True) + # hash = pluginapi.get_core().insertBlock(, header = 'plugin', sign = True) if import_block: pluginapi.get_utils().importNewBlocks() @@ -226,7 +248,7 @@ def pluginToBlock(plugin, import_block = True): def installBlock(block): try: - block = Block(block) + block = Block(block, core = pluginapi.get_core()) blockContent = json.loads(block.getContent()) name = sanitize(blockContent['name']) @@ -353,7 +375,8 @@ def commandInstallPlugin(): choice = logger.readline('Select the number of the key to use, from 1 to %s, or press Ctrl+C to cancel:' % (index - 1)) try: - if int(choice) < index and int(choice) >= 1: + choice = int(choice) + if choice <= index and choice >= 1: distributor = distributors[int(choice)] valid = True except KeyboardInterrupt: @@ -367,10 +390,12 @@ def commandInstallPlugin(): except Exception as e: logger.warn('Failed to lookup plugin in repositories.', timestamp = False) logger.error('asdf', error = e, timestamp = False) + + return True if pkobh is None: - logger.error('No key for this plugin found in keystore or repositories, please specify.') - help() + logger.error('No key for this plugin found in keystore or repositories, please specify.', timestamp = False) + return True valid_hash = pluginapi.get_utils().validateHash(pkobh) @@ -386,7 +411,7 @@ def commandInstallPlugin(): blockhash = None if valid_hash and not real_block: - logger.error('Block hash not found. Perhaps it has not been synced yet?') + logger.error('Block hash not found. Perhaps it has not been synced yet?', timestamp = False) logger.debug('Is valid hash, but does not belong to a known block.') return True @@ -396,7 +421,7 @@ def commandInstallPlugin(): installBlock(blockhash) elif valid_key and not real_key: - logger.error('Public key not found. Try adding the node by address manually, if possible.') + logger.error('Public key not found. Try adding the node by address manually, if possible.', timestamp = False) logger.debug('Is valid key, but the key is not a known one.') elif valid_key and real_key: publickey = str(pkobh) @@ -432,10 +457,11 @@ def commandInstallPlugin(): except Exception as e: pass - logger.warn('Only continue the installation is you are absolutely certain that you trust the plugin distributor. Public key of plugin distributor: %s' % publickey, timestamp = False) + logger.warn('Only continue the installation if you are absolutely certain that you trust the plugin distributor. Public key of plugin distributor: %s' % publickey, timestamp = False) + logger.debug('Most recent block matching parameters is %s' % mostRecentVersionBlock) installBlock(mostRecentVersionBlock) else: - logger.error('Unknown data "%s"; must be public key or block hash.' % str(pkobh)) + logger.error('Unknown data "%s"; must be public key or block hash.' % str(pkobh), timestamp = False) return else: logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' [public key/block hash]') @@ -463,11 +489,11 @@ def commandAddRepository(): if pluginapi.get_utils().validateHash(blockhash): if Block.exists(blockhash): try: - blockContent = json.loads(Block(blockhash).getContent()) + blockContent = json.loads(Block(blockhash, core = pluginapi.get_core()).getContent()) pluginslist = dict() - for pluginname, distributor in blockContent['plugins'].items(): + for pluginname, distributor in blockContent['plugins']: if pluginapi.get_utils().validatePubKey(distributor): pluginslist[pluginname] = distributor @@ -477,14 +503,14 @@ def commandAddRepository(): addRepository(blockhash, pluginslist) logger.info('Successfully added repository.') else: - logger.error('Repository contains no records, not importing.') + logger.error('Repository contains no records, not importing.', timestamp = False) except Exception as e: logger.error('Failed to parse block.', error = e) else: - logger.error('Block hash not found. Perhaps it has not been synced yet?') + logger.error('Block hash not found. Perhaps it has not been synced yet?', timestamp = False) logger.debug('Is valid hash, but does not belong to a known block.') else: - logger.error('Unknown data "%s"; must be block hash.' % str(pkobh)) + logger.error('Unknown data "%s"; must be block hash.' % str(pkobh), timestamp = False) else: logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' [block hash]') @@ -500,10 +526,11 @@ def commandRemoveRepository(): if blockhash in getRepositories(): try: removeRepository(blockhash) + logger.info('Successfully removed repository.') except Exception as e: logger.error('Failed to parse block.', error = e) else: - logger.error('Repository has not been imported, nothing to remove.') + logger.error('Repository has not been imported, nothing to remove.', timestamp = False) else: logger.error('Unknown data "%s"; must be block hash.' % str(pkobh)) else: @@ -525,7 +552,49 @@ def commandPublishPlugin(): logger.error('Plugin %s does not exist.' % pluginname, timestamp = False) else: logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' ') - + +def commandCreateRepository(): + if len(sys.argv) >= 3: + check() + + plugins = list() + script = sys.argv[0] + + del sys.argv[:2] + success = True + for pluginname in sys.argv: + distributor = None + + if ':' in pluginname: + split = pluginname.split(':') + pluginname = split[0] + distributor = split[1] + + pluginname = sanitize(pluginname) + + if distributor is None: + distributor = getKey(pluginname) + if distributor is None: + logger.error('No distributor key was found for the plugin %s.' % pluginname, timestamp = False) + success = False + + plugins.append([pluginname, distributor]) + + if not success: + logger.error('Please correct the above errors, then recreate the repository.') + return True + + blockhash = createRepository(plugins) + print(blockhash) + if not blockhash is None: + logger.info('Successfully created repository. Execute the following command to add the repository:\n ' + logger.colors.underline + '%s --add-repository %s' % (script, blockhash)) + else: + logger.error('Failed to create repository, an unknown error occurred.') + else: + logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' [plugins...]') + + return True + # event listeners def on_init(api, data = None): @@ -540,6 +609,7 @@ def on_init(api, data = None): api.commands.register(['add-repo', 'add-repository', 'addrepo', 'addrepository', 'repository-add', 'repo-add', 'repoadd', 'addrepository', 'add-plugin-repository', 'add-plugin-repo', 'add-pluginrepo', 'add-pluginrepository', 'addpluginrepo', 'addpluginrepository'], commandAddRepository) api.commands.register(['remove-repo', 'remove-repository', 'removerepo', 'removerepository', 'repository-remove', 'repo-remove', 'reporemove', 'removerepository', 'remove-plugin-repository', 'remove-plugin-repo', 'remove-pluginrepo', 'remove-pluginrepository', 'removepluginrepo', 'removepluginrepository', 'rm-repo', 'rm-repository', 'rmrepo', 'rmrepository', 'repository-rm', 'repo-rm', 'reporm', 'rmrepository', 'rm-plugin-repository', 'rm-plugin-repo', 'rm-pluginrepo', 'rm-pluginrepository', 'rmpluginrepo', 'rmpluginrepository'], commandRemoveRepository) api.commands.register(['publish-plugin', 'plugin-publish', 'publishplugin', 'pluginpublish', 'publish'], commandPublishPlugin) + api.commands.register(['create-repository', 'create-repo', 'createrepo', 'createrepository', 'repocreate'], commandCreateRepository) # add help menus once the features are actually implemented