kev/Onionr
1
0
Fork 0
Code Issues 24 Pull Requests Projects Releases 2 Wiki Activity

See details

Browse Source 
- Completes support for repositories
  - `./RUN-LINUX.sh create-repository [plugins...]`
  - `./RUN-LINUX.sh add-repository <block hash>`
  - `./RUN-LINUX.sh remove-repository <block hash>`
- Fixes several misc bugs
- Refactors code
  - Some messy code was rewritten
  - Variables renamed
  - Migrated old block api (insertBlock) to new Block API (onionrblockapi)
  - Kept to standards
  - Made code more reusable in `onionrproofs.py`
- Improves logging messages
  - Added error output for some features missing it
  - Capitalized sentences
  - Added punctuation where it is missing
  - Switched `logger.info` and `logger.debug` in a few places, where it is logical
  - Removed or added timestamps depending on the circumstance
- Added a few misc features
  - Added command aliases for `add-file` and `import-blocks`
  - Improved statistics menu
    - Displays `Known Block Count`
    - Calculates and displays `Percent Blocks Signed`
This commit is contained in:
Arinerron 2018-05-31 21:25:28 -07:00
parent a232e663a7
commit 8846dcc2c6
No known key found for this signature in database
GPG Key ID: 99383627861C62F0
7 changed files with 184 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
onionr/api.py
View File

@ -70,7 +70,7 @@ class API:
self.clientToken = config.get('client')['client_hmac']
self.timeBypassToken = base64.b16encode(os.urandom(32)).decode()
self.i2pEnabled = config.get('i2p')['host']
self.i2pEnabled = config.get('i2p', {'host' : False})['host']
self.mimeType = 'text/plain'
@ -85,9 +85,9 @@ class API:
self.host = '127.' + str(hostNums[0]) + '.' + str(hostNums[1]) + '.' + str(hostNums[2])
else:
self.host = '127.0.0.1'
hostFile = open('data/host.txt', 'w')
hostFile.write(self.host)
hostFile.close()
with open('data/host.txt', 'w') as file:
file.write(self.host)
@app.before_request
def beforeReq():
@ -259,7 +259,7 @@ class API:
return resp
if not os.environ.get("WERKZEUG_RUN_MAIN") == "true":
logger.info('Starting client on ' + self.host + ':' + str(bindPort) + '...', timestamp=True)
logger.info('Starting client on ' + self.host + ':' + str(bindPort) + '...', timestamp=False)
try:
self.http_server = WSGIServer((self.host, bindPort), app)

23
onionr/communicator.py
View File

@ -123,16 +123,16 @@ class OnionrCommunicate:
announceAttempts = 3
announceAttemptCount = 0
announceVal = False
logger.info('Announcing node to ' + command[1], timestamp=True)
logger.info('Announcing node to %s...' % command[1], timestamp=True)
while not announceVal:
announceAttemptCount += 1
announceVal = self.performGet('announce', command[1], data=self._core.hsAdder.replace('\n', ''), skipHighFailureAddress=True)
logger.info(announceVal)
# logger.info(announceVal)
if announceAttemptCount >= announceAttempts:
logger.warn('Unable to announce to ' + command[1])
logger.warn('Unable to announce to %s' % command[1])
break
elif command[0] == 'runCheck':
logger.info('Status check; looks good.')
logger.debug('Status check; looks good.')
open('data/.runcheck', 'w+').close()
elif command[0] == 'kex':
self.pexCount = pexTimer - 1
@ -188,13 +188,17 @@ class OnionrCommunicate:
id_peer_cache = {}
def registerTimer(self, timerName, rate, timerFunc=None):
'''Register a communicator timer'''
'''
Register a communicator timer
'''
self.communicatorTimers[timerName] = rate
self.communicatorTimerCounts[timerName] = 0
self.communicatorTimerFuncs[timerName] = timerFunc
def timerTick(self):
'''Increments timers "ticks" and calls funcs if applicable'''
'''
Increments timers "ticks" and calls funcs if applicable
'''
tName = ''
for i in self.communicatorTimers.items():
tName = i[0]
@ -617,7 +621,9 @@ class OnionrCommunicate:
return
def removeBlockFromProcessingList(self, block):
'''Remove a block from the processing list'''
'''
Remove a block from the processing list
'''
try:
self.blocksProcessing.remove(block)
except ValueError:
@ -724,7 +730,8 @@ class OnionrCommunicate:
r = requests.get(url, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30))
retData = r.text
except requests.exceptions.RequestException as e:
logger.debug("%s failed with peer %s" % (action, peer))
logger.debug('%s failed with peer %s' % (action, peer))
logger.debug('Error: %s' % str(e))
retData = False
if not retData:

2
onionr/netcontroller.py
View File

@ -103,7 +103,7 @@ DataDirectory data/tordata/
logger.fatal("Got keyboard interrupt")
return False
logger.info('Finished starting Tor', timestamp=True)
logger.debug('Finished starting Tor.', timestamp=True)
self.readyState = True
myID = open('data/hs/hostname', 'r')

60
onionr/onionr.py
View File

@ -31,6 +31,7 @@ import api, core, config, logger, onionrplugins as plugins, onionrevents as even
import onionrutils
from onionrutils import OnionrUtils
from netcontroller import NetController
from onionrblockapi import Block
try:
from urllib3.contrib.socks import SOCKSProxyManager
@ -192,8 +193,11 @@ class Onionr:
'add-addr': self.addAddress,
'addaddr': self.addAddress,
'addaddress': self.addAddress,
'add-file': self.addFile,
'addfile': self.addFile,
'import-blocks': self.onionrUtils.importNewBlocks,
'importblocks': self.onionrUtils.importNewBlocks,
'introduce': self.onionrCore.introduceNode,
@ -216,8 +220,8 @@ class Onionr:
'add-msg': 'Broadcasts a message to the Onionr network',
'pm': 'Adds a private message to block',
'get-pms': 'Shows private messages sent to you',
'addfile': 'Create an Onionr block from a file',
'importblocks': 'import blocks from the disk (Onionr is transport-agnostic!)',
'add-file': 'Create an Onionr block from a file',
'import-blocks': 'import blocks from the disk (Onionr is transport-agnostic!)',
'introduce': 'Introduce your node to the public Onionr network',
}
@ -391,12 +395,11 @@ class Onionr:
except KeyboardInterrupt:
return
#addedHash = self.onionrCore.setData(messageToAdd)
addedHash = self.onionrCore.insertBlock(messageToAdd, header='txt')
#self.onionrCore.addToBlockDB(addedHash, selfInsert=True)
#self.onionrCore.setBlockType(addedHash, 'txt')
if addedHash != '':
addedHash = Block('txt', messageToAdd).save()
if addedHash != None:
logger.info("Message inserted as as block %s" % addedHash)
else:
logger.error('Failed to insert block.', timestamp = False)
return
def getPMs(self):
@ -520,12 +523,12 @@ class Onionr:
if not os.environ.get("WERKZEUG_RUN_MAIN") == "true":
if self._developmentMode:
logger.warn('DEVELOPMENT MODE ENABLED (THIS IS LESS SECURE!)')
logger.warn('DEVELOPMENT MODE ENABLED (THIS IS LESS SECURE!)', timestamp = False)
net = NetController(config.get('client')['port'])
logger.info('Tor is starting...')
if not net.startTor():
sys.exit(1)
logger.info('Started Tor .onion service: ' + logger.colors.underline + net.myID)
logger.info('Started .onion service: ' + logger.colors.underline + net.myID)
logger.info('Our Public key: ' + self.onionrCore._crypto.pubKey)
time.sleep(1)
subprocess.Popen(["./communicator.py", "run", str(net.socksPort)])
@ -562,6 +565,9 @@ class Onionr:
try:
# define stats messages here
totalBlocks = len(Block.getBlocks())
signedBlocks = len(Block.getBlocks(signed = True))
messages = {
# info about local client
'Onionr Daemon Status' : ((logger.colors.fg.green + 'Online') if self.onionrUtils.isCommunicatorRunning(timeout = 2) else logger.colors.fg.red + 'Offline'),
@ -577,7 +583,9 @@ class Onionr:
# count stats
'div2' : True,
'Known Peers Count' : str(len(self.onionrCore.listPeers()) - 1),
'Enabled Plugins Count' : str(len(config.get('plugins')['enabled'])) + ' / ' + str(len(os.listdir('data/plugins/')))
'Enabled Plugins Count' : str(len(config.get('plugins')['enabled'])) + ' / ' + str(len(os.listdir('data/plugins/'))),
'Known Blocks Count' : str(totalBlocks),
'Percent Blocks Signed' : str(round(100 * signedBlocks / totalBlocks, 2)) + '%'
}
# color configuration
@ -639,18 +647,30 @@ class Onionr:
return None
def addFile(self):
'''command to add a file to the onionr network'''
if len(sys.argv) >= 2:
newFile = sys.argv[2]
logger.info('Attempting to add file...')
try:
with open(newFile, 'rb') as new:
new = new.read()
except FileNotFoundError:
'''
Adds a file to the onionr network
'''
if len(sys.argv) >= 3:
filename = sys.argv[2]
contents = None
if not os.path.exists(filename):
logger.warn('That file does not exist. Improper path?')
try:
with open(filename, 'rb') as file:
contents = file.read().decode()
except:
pass
if not contents is None:
blockhash = Block('bin', contents).save()
logger.info('File %s saved in block %s.' % (filename, blockhash))
else:
logger.debug(new)
logger.info(self.onionrCore.insertBlock(new, header='bin'))
logger.error('Failed to save file in block.', timestamp = False)
else:
logger.error('%s add-file <filename>' % sys.argv[0], timestamp = False)
Onionr()

10
onionr/onionrplugins.py
View File

@ -77,7 +77,7 @@ def enable(name, onionr = None, start_event = True):
else:
return False
else:
logger.error('Failed to enable plugin \"' + name + '\", disabling plugin.')
logger.error('Failed to enable plugin \"%s\", disabling plugin.' % name)
disable(name)
return False
@ -121,9 +121,9 @@ def start(name, onionr = None):
return plugin
except:
logger.error('Failed to start module \"' + name + '\".')
logger.error('Failed to start module \"%s\".' % name)
else:
logger.error('Failed to start nonexistant module \"' + name + '\".')
logger.error('Failed to start nonexistant module \"%s\".' % name)
return None
@ -145,9 +145,9 @@ def stop(name, onionr = None):
return plugin
except:
logger.error('Failed to stop module \"' + name + '\".')
logger.error('Failed to stop module \"%s\".' % name)
else:
logger.error('Failed to stop nonexistant module \"' + name + '\".')
logger.error('Failed to stop nonexistant module \"%s\".' % name)
return None

39
onionr/onionrproofs.py
View File

@ -22,7 +22,7 @@ import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, lo
import core
class POW:
def pow(self, reporting = False):
def pow(self, reporting = False, myCore = None):
startTime = math.floor(time.time())
self.hashing = True
self.reporting = reporting
@ -30,7 +30,7 @@ class POW:
answer = ''
heartbeat = 200000
hbCount = 0
myCore = core.Core()
while self.hashing:
rand = nacl.utils.random()
token = nacl.hash.blake2b(rand + self.data).decode()
@ -39,23 +39,22 @@ class POW:
self.hashing = False
iFound = True
break
else:
logger.debug('POW thread exiting, another thread found result')
if iFound:
endTime = math.floor(time.time())
if self.reporting:
logger.info('Found token ' + token, timestamp=True)
logger.info('rand value: ' + base64.b64encode(rand).decode())
logger.info('took ' + str(endTime - startTime) + ' seconds', timestamp=True)
logger.debug('Found token after %s seconds: %s' % (endTime - startTime, token), timestamp=True)
logger.debug('Random value was: %s' % base64.b64encode(rand).decode())
self.result = (token, rand)
def __init__(self, data):
def __init__(self, data, threadCount = 5):
self.foundHash = False
self.difficulty = 0
self.data = data
self.threadCount = threadCount
dataLen = sys.getsizeof(data)
self.difficulty = math.floor(dataLen/1000000)
self.difficulty = math.floor(dataLen / 1000000)
if self.difficulty <= 2:
self.difficulty = 4
@ -63,19 +62,19 @@ class POW:
self.data = self.data.encode()
except AttributeError:
pass
self.data = nacl.hash.blake2b(self.data)
logger.debug('Computing difficulty of ' + str(self.difficulty))
logger.info('Computing POW (difficulty: %s)...' % self.difficulty)
self.mainHash = '0' * 70
self.puzzle = self.mainHash[0:min(self.difficulty, len(self.mainHash))]
myCore = core.Core()
for i in range(max(1, threadCount)):
t = threading.Thread(name = 'thread%s' % i, target = self.pow, args = (True,myCore))
t.start()
self.mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode()
self.puzzle = self.mainHash[0:self.difficulty]
#logger.debug('trying to find ' + str(self.mainHash))
tOne = threading.Thread(name='one', target=self.pow, args=(True,))
tTwo = threading.Thread(name='two', target=self.pow, args=(True,))
tThree = threading.Thread(name='three', target=self.pow, args=(True,))
tOne.start()
tTwo.start()
tThree.start()
return
def shutdown(self):
@ -89,9 +88,11 @@ class POW:
'''
Returns the result then sets to false, useful to automatically clear the result
'''
try:
retVal = self.result
except AttributeError:
retVal = False
self.result = False
return retVal

124
onionr/static-data/default-plugins/pluginmanager/main.py
View File

@ -34,9 +34,9 @@ def writeKeys():
Serializes and writes the keystore in memory to file
'''
file = open(keys_file, 'w')
file.write(json.dumps(keys_data, indent=4, sort_keys=True))
file.close()
with open(keys_file, 'w') as file:
file.write(json.dumps(keys_data, indent=4, sort_keys=True))
file.close()
def readKeys():
'''
@ -44,7 +44,8 @@ def readKeys():
'''
global keys_data
keys_data = json.loads(open(keys_file).read())
with open(keys_file) as file:
keys_data = json.loads(file.read())
return keys_data
def getKey(plugin):
@ -106,27 +107,37 @@ def getRepositories():
readKeys()
return keys_data['repositories']
def addRepository(repositories, data):
def addRepository(blockhash, data):
'''
Saves the plugin name, to remember that it was installed by the pluginmanager
'''
global keys_data
readKeys()
keys_data['repositories'][repositories] = data
keys_data['repositories'][blockhash] = data
writeKeys()
def removeRepository(repositories):
def removeRepository(blockhash):
'''
Removes the plugin name from the pluginmanager's records
'''
global keys_data
readKeys()
if plugin in keys_data['repositories']:
del keys_data['repositories'][repositories]
if blockhash in keys_data['repositories']:
del keys_data['repositories'][blockhash]
writeKeys()
def createRepository(plugins):
contents = {'plugins' : plugins, 'author' : getpass.getuser(), 'compiled-by' : plugin_name}
block = Block(core = pluginapi.get_core())
block.setType('repository')
block.setContent(json.dumps(contents))
return block.save(True)
def check():
'''
Checks to make sure the keystore file still exists
@ -144,7 +155,7 @@ def sanitize(name):
def blockToPlugin(block):
try:
block = Block(block)
block = Block(block, core = pluginapi.get_core())
blockContent = json.loads(block.getContent())
name = sanitize(blockContent['name'])
@ -194,14 +205,19 @@ def pluginToBlock(plugin, import_block = True):
shutil.rmtree(directory + '__pycache__')
shutil.make_archive(zipfile[:-4], 'zip', directory)
data = base64.b64encode(open(zipfile, 'rb').read())
data = ''
with open(zipfile, 'rb') as file:
data = base64.b64encode(file.read())
author = getpass.getuser()
description = 'Default plugin description'
info = {"name" : plugin}
try:
if os.path.exists(directory + 'info.json'):
info = json.loads(open(directory + 'info.json').read())
info = ''
with open(directory + 'info.json').read() as file:
info = json.loads(file.read())
if 'author' in info:
author = info['author']
if 'description' in info:
@ -211,7 +227,13 @@ def pluginToBlock(plugin, import_block = True):
metadata = {'author' : author, 'date' : str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')), 'name' : plugin, 'info' : info, 'compiled-by' : plugin_name, 'content' : data.decode('utf-8'), 'description' : description}
hash = pluginapi.get_core().insertBlock(json.dumps(metadata), header = 'plugin', sign = True)
block = Block(core = pluginapi.get_core())
block.setType('plugin')
block.setContent(json.dumps(metadata))
hash = block.save(True)
# hash = pluginapi.get_core().insertBlock(, header = 'plugin', sign = True)
if import_block:
pluginapi.get_utils().importNewBlocks()
@ -226,7 +248,7 @@ def pluginToBlock(plugin, import_block = True):
def installBlock(block):
try:
block = Block(block)
block = Block(block, core = pluginapi.get_core())
blockContent = json.loads(block.getContent())
name = sanitize(blockContent['name'])
@ -353,7 +375,8 @@ def commandInstallPlugin():
choice = logger.readline('Select the number of the key to use, from 1 to %s, or press Ctrl+C to cancel:' % (index - 1))
try:
if int(choice) < index and int(choice) >= 1:
choice = int(choice)
if choice <= index and choice >= 1:
distributor = distributors[int(choice)]
valid = True
except KeyboardInterrupt:
@ -368,9 +391,11 @@ def commandInstallPlugin():
logger.warn('Failed to lookup plugin in repositories.', timestamp = False)
logger.error('asdf', error = e, timestamp = False)
return True
if pkobh is None:
logger.error('No key for this plugin found in keystore or repositories, please specify.')
help()
logger.error('No key for this plugin found in keystore or repositories, please specify.', timestamp = False)
return True
valid_hash = pluginapi.get_utils().validateHash(pkobh)
@ -386,7 +411,7 @@ def commandInstallPlugin():
blockhash = None
if valid_hash and not real_block:
logger.error('Block hash not found. Perhaps it has not been synced yet?')
logger.error('Block hash not found. Perhaps it has not been synced yet?', timestamp = False)
logger.debug('Is valid hash, but does not belong to a known block.')
return True
@ -396,7 +421,7 @@ def commandInstallPlugin():
installBlock(blockhash)
elif valid_key and not real_key:
logger.error('Public key not found. Try adding the node by address manually, if possible.')
logger.error('Public key not found. Try adding the node by address manually, if possible.', timestamp = False)
logger.debug('Is valid key, but the key is not a known one.')
elif valid_key and real_key:
publickey = str(pkobh)
@ -432,10 +457,11 @@ def commandInstallPlugin():
except Exception as e:
pass
logger.warn('Only continue the installation is you are absolutely certain that you trust the plugin distributor. Public key of plugin distributor: %s' % publickey, timestamp = False)
logger.warn('Only continue the installation if you are absolutely certain that you trust the plugin distributor. Public key of plugin distributor: %s' % publickey, timestamp = False)
logger.debug('Most recent block matching parameters is %s' % mostRecentVersionBlock)
installBlock(mostRecentVersionBlock)
else:
logger.error('Unknown data "%s"; must be public key or block hash.' % str(pkobh))
logger.error('Unknown data "%s"; must be public key or block hash.' % str(pkobh), timestamp = False)
return
else:
logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin> [public key/block hash]')
@ -463,11 +489,11 @@ def commandAddRepository():
if pluginapi.get_utils().validateHash(blockhash):
if Block.exists(blockhash):
try:
blockContent = json.loads(Block(blockhash).getContent())
blockContent = json.loads(Block(blockhash, core = pluginapi.get_core()).getContent())
pluginslist = dict()
for pluginname, distributor in blockContent['plugins'].items():
for pluginname, distributor in blockContent['plugins']:
if pluginapi.get_utils().validatePubKey(distributor):
pluginslist[pluginname] = distributor
@ -477,14 +503,14 @@ def commandAddRepository():
addRepository(blockhash, pluginslist)
logger.info('Successfully added repository.')
else:
logger.error('Repository contains no records, not importing.')
logger.error('Repository contains no records, not importing.', timestamp = False)
except Exception as e:
logger.error('Failed to parse block.', error = e)
else:
logger.error('Block hash not found. Perhaps it has not been synced yet?')
logger.error('Block hash not found. Perhaps it has not been synced yet?', timestamp = False)
logger.debug('Is valid hash, but does not belong to a known block.')
else:
logger.error('Unknown data "%s"; must be block hash.' % str(pkobh))
logger.error('Unknown data "%s"; must be block hash.' % str(pkobh), timestamp = False)
else:
logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' [block hash]')
@ -500,10 +526,11 @@ def commandRemoveRepository():
if blockhash in getRepositories():
try:
removeRepository(blockhash)
logger.info('Successfully removed repository.')
except Exception as e:
logger.error('Failed to parse block.', error = e)
else:
logger.error('Repository has not been imported, nothing to remove.')
logger.error('Repository has not been imported, nothing to remove.', timestamp = False)
else:
logger.error('Unknown data "%s"; must be block hash.' % str(pkobh))
else:
@ -526,6 +553,48 @@ def commandPublishPlugin():
else:
logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin>')
def commandCreateRepository():
if len(sys.argv) >= 3:
check()
plugins = list()
script = sys.argv[0]
del sys.argv[:2]
success = True
for pluginname in sys.argv:
distributor = None
if ':' in pluginname:
split = pluginname.split(':')
pluginname = split[0]
distributor = split[1]
pluginname = sanitize(pluginname)
if distributor is None:
distributor = getKey(pluginname)
if distributor is None:
logger.error('No distributor key was found for the plugin %s.' % pluginname, timestamp = False)
success = False
plugins.append([pluginname, distributor])
if not success:
logger.error('Please correct the above errors, then recreate the repository.')
return True
blockhash = createRepository(plugins)
print(blockhash)
if not blockhash is None:
logger.info('Successfully created repository. Execute the following command to add the repository:\n ' + logger.colors.underline + '%s --add-repository %s' % (script, blockhash))
else:
logger.error('Failed to create repository, an unknown error occurred.')
else:
logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' [plugins...]')
return True
# event listeners
def on_init(api, data = None):
@ -540,6 +609,7 @@ def on_init(api, data = None):
api.commands.register(['add-repo', 'add-repository', 'addrepo', 'addrepository', 'repository-add', 'repo-add', 'repoadd', 'addrepository', 'add-plugin-repository', 'add-plugin-repo', 'add-pluginrepo', 'add-pluginrepository', 'addpluginrepo', 'addpluginrepository'], commandAddRepository)
api.commands.register(['remove-repo', 'remove-repository', 'removerepo', 'removerepository', 'repository-remove', 'repo-remove', 'reporemove', 'removerepository', 'remove-plugin-repository', 'remove-plugin-repo', 'remove-pluginrepo', 'remove-pluginrepository', 'removepluginrepo', 'removepluginrepository', 'rm-repo', 'rm-repository', 'rmrepo', 'rmrepository', 'repository-rm', 'repo-rm', 'reporm', 'rmrepository', 'rm-plugin-repository', 'rm-plugin-repo', 'rm-pluginrepo', 'rm-pluginrepository', 'rmpluginrepo', 'rmpluginrepository'], commandRemoveRepository)
api.commands.register(['publish-plugin', 'plugin-publish', 'publishplugin', 'pluginpublish', 'publish'], commandPublishPlugin)
api.commands.register(['create-repository', 'create-repo', 'createrepo', 'createrepository', 'repocreate'], commandCreateRepository)
# add help menus once the features are actually implemented