kev/Onionr
1
0
Fork 0
Code Issues 24 Pull Requests Projects Releases 2 Wiki Activity

Merge branch 'communicator-multithreading' of github.com:beardog108/onionr into communicator-multithreading

Browse Source
This commit is contained in:
Kevin Froman 2018-05-14 11:32:01 -05:00
commit 361f1e3d54
12 changed files with 525 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
onionr/communicator.py
View File

@ -447,7 +447,7 @@ class OnionrCommunicate:
if isThread: if isThread:
self.lookupBlocksThreads += 1 self.lookupBlocksThreads += 1
peerList = self._core.listAdders() peerList = self._core.listAdders()
blocks = '' blockList = list()
for i in peerList: for i in peerList:
if self.peerStatusTaken(i, 'getBlockHashes') or self.peerStatusTaken(i, 'getDBHash'): if self.peerStatusTaken(i, 'getBlockHashes') or self.peerStatusTaken(i, 'getDBHash'):
@ -476,18 +476,15 @@ class OnionrCommunicate:
if lastDB != currentDB: if lastDB != currentDB:
logger.debug('Fetching hash from %s - %s current hash.' % (str(i), currentDB)) logger.debug('Fetching hash from %s - %s current hash.' % (str(i), currentDB))
try: try:
blocks += self.performGet('getBlockHashes', i) blockList.append(self.performGet('getBlockHashes', i))
except TypeError: except TypeError:
logger.warn('Failed to get data hash from %s' % str(i)) logger.warn('Failed to get data hash from %s' % str(i))
self.peerData[i]['failCount'] -= 1 self.peerData[i]['failCount'] -= 1
if self._utils.validateHash(currentDB): if self._utils.validateHash(currentDB):
self._core.setAddressInfo(i, "DBHash", currentDB) self._core.setAddressInfo(i, "DBHash", currentDB)
if len(blocks.strip()) != 0: if len(blockList) != 0:
pass pass
#logger.debug('BLOCKS:' + blocks)
blockList = blocks.split('\n')
for i in blockList: for i in blockList:
if len(i.strip()) == 0: if len(i.strip()) == 0:
@ -517,7 +514,7 @@ class OnionrCommunicate:
''' '''
if isThread: if isThread:
self.processBlocksThreads += 1 self.processBlocksThreads += 1
for i in self._core.getBlockList(unsaved=True).split("\n"): for i in self._core.getBlockList(unsaved = True):
if i != "": if i != "":
if i in self.blocksProcessing or i in self.ignoredHashes: if i in self.blocksProcessing or i in self.ignoredHashes:
#logger.debug('already processing ' + i) #logger.debug('already processing ' + i)
@ -553,43 +550,39 @@ class OnionrCommunicate:
pass pass
try: try:
#blockMetadata = json.loads(self._core.getData(i)).split('}')[0] + '}' #blockMetadata = json.loads(self._core.getData(i)).split('}')[0] + '}'
blockMetadata = json.loads(blockContent[:blockContent.rfind(b'}') + 1].decode()) blockMetadata = json.loads(blockContent[:blockContent.find(b'\n')].decode())
try: try:
blockMeta2 = json.loads(blockMetadata['meta']) blockMeta2 = json.loads(blockMetadata['meta'])
except KeyError: except KeyError:
blockMeta2 = {'type': ''} blockMeta2 = {'type': ''}
pass pass
blockContent = blockContent[blockContent.rfind(b'}') + 1:] blockContent = blockContent[blockContent.find(b'\n') + 1:]
try: try:
blockContent = blockContent.decode() blockContent = blockContent.decode()
except AttributeError: except AttributeError:
pass pass
if not self.verifyPow(blockContent, blockMeta2): if not self._crypto.verifyPow(blockContent, blockMeta2):
logger.warn("%s has invalid or insufficient proof of work token, deleting..." % str(i)) logger.warn("%s has invalid or insufficient proof of work token, deleting..." % str(i))
self._core.removeBlock(i) self._core.removeBlock(i)
continue continue
try:
blockMetadata['sig']
blockMeta2['id']
except KeyError:
pass
else: else:
#blockData = json.dumps(blockMetadata['meta']) + blockMetadata[blockMetadata.rfind(b'}') + 1:] if (('sig' in blockMetadata) and ('id' in blockMeta2)): # id doesn't exist in blockMeta2, so this won't workin the first place
#blockData = json.dumps(blockMetadata['meta']) + blockMetadata[blockMetadata.rfind(b'}') + 1:]
creator = self._utils.getPeerByHashId(blockMeta2['id']) creator = self._utils.getPeerByHashId(blockMeta2['id'])
try: try:
creator = creator.decode() creator = creator.decode()
except AttributeError: except AttributeError:
pass pass
if self._core._crypto.edVerify(blockMetaData['meta'] + blockContent, creator, blockMetadata['sig'], encodedData=True): if self._core._crypto.edVerify(blockMetadata['meta'] + blockContent, creator, blockMetadata['sig'], encodedData=True):
logger.info('%s was signed' % str(i)) logger.info('%s was signed' % str(i))
self._core.updateBlockInfo(i, 'sig', 'true') self._core.updateBlockInfo(i, 'sig', 'true')
else: else:
logger.warn('%s has an invalid signature' % str(i)) logger.warn('%s has an invalid signature' % str(i))
self._core.updateBlockInfo(i, 'sig', 'false') self._core.updateBlockInfo(i, 'sig', 'false')
try: try:
logger.info('Block type is %s' % str(blockMeta2['type'])) logger.info('Block type is %s' % str(blockMeta2['type']))
self._core.updateBlockInfo(i, 'dataType', blockMeta2['type']) self._core.updateBlockInfo(i, 'dataType', blockMeta2['type'])
@ -605,12 +598,7 @@ class OnionrCommunicate:
return return
def removeBlockFromProcessingList(self, block): def removeBlockFromProcessingList(self, block):
try: return block in blocksProcessing
self.blocksProcessing.remove(block)
except ValueError:
return False
else:
return True
def downloadBlock(self, hash, peerTries=3): def downloadBlock(self, hash, peerTries=3):
''' '''
@ -666,41 +654,6 @@ class OnionrCommunicate:
return retVal return retVal
def verifyPow(self, blockContent, metadata):
'''
Verifies the proof of work associated with a block
'''
retData = False
try:
metadata['powToken']
metadata['powHash']
token = metadata['powToken']
except KeyError:
return False
dataLen = len(blockContent)
expectedHash = self._crypto.blake2bHash(base64.b64decode(metadata['powToken']) + self._crypto.blake2bHash(blockContent.encode()))
difficulty = 0
try:
expectedHash = expectedHash.decode()
except AttributeError:
pass
if metadata['powHash'] == expectedHash:
difficulty = math.floor(dataLen/1000000)
mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode()
puzzle = mainHash[0:difficulty]
if metadata['powHash'][0:difficulty] == puzzle:
logger.info('Validated block pow')
retData = True
else:
logger.warn("Invalid token (#1)")
else:
logger.warn('Invalid token (#2): Expected hash %s, got hash %s...' % (metadata['powHash'], expectedHash))
return retData
def urlencode(self, data): def urlencode(self, data):
''' '''
URL encodes the data URL encodes the data

21
onionr/core.py
View File

@ -277,11 +277,12 @@ class Core:
return return
def getData(self,hash): def getData(self, hash):
''' '''
Simply return the data associated to a hash Simply return the data associated to a hash
''' '''
try: try:
# logger.debug('Opening %s' % (str(self.blockDataLocation) + str(hash) + '.dat'))
dataFile = open(self.blockDataLocation + hash + '.dat', 'rb') dataFile = open(self.blockDataLocation + hash + '.dat', 'rb')
data = dataFile.read() data = dataFile.read()
dataFile.close() dataFile.close()
@ -576,22 +577,22 @@ class Core:
return return
def getBlockList(self, unsaved = False): def getBlockList(self, unsaved = False): # TODO: Use unsaved
''' '''
Get list of our blocks Get list of our blocks
''' '''
conn = sqlite3.connect(self.blockDB) conn = sqlite3.connect(self.blockDB)
c = conn.cursor() c = conn.cursor()
retData = ''
if unsaved: if unsaved:
execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();' execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();'
else: else:
execute = 'SELECT hash FROM hashes ORDER BY RANDOM();' execute = 'SELECT hash FROM hashes ORDER BY RANDOM();'
rows = list()
for row in c.execute(execute): for row in c.execute(execute):
for i in row: for i in row:
retData += i + "\n" rows.append(i)
return retData return rows
def getBlocksByType(self, blockType): def getBlocksByType(self, blockType):
''' '''
@ -599,14 +600,14 @@ class Core:
''' '''
conn = sqlite3.connect(self.blockDB) conn = sqlite3.connect(self.blockDB)
c = conn.cursor() c = conn.cursor()
retData = ''
execute = 'SELECT hash FROM hashes WHERE dataType=?;' execute = 'SELECT hash FROM hashes WHERE dataType=?;'
args = (blockType,) args = (blockType,)
rows = list()
for row in c.execute(execute, args): for row in c.execute(execute, args):
for i in row: for i in row:
retData += i + "\n" rows.append(i)
return retData.split('\n') return rows
def setBlockType(self, hash, blockType): def setBlockType(self, hash, blockType):
''' '''
@ -677,7 +678,7 @@ class Core:
signature = '' signature = ''
if sign: if sign:
signature = self._crypto.edSign(metadata + data, self._crypto.privKey, encodeResult=True) signature = self._crypto.edSign(metadata + b'\n' + data, self._crypto.privKey, encodeResult=True)
ourID = self._crypto.pubKeyHashID() ourID = self._crypto.pubKeyHashID()
# Convert from bytes on some py versions? # Convert from bytes on some py versions?
try: try:
@ -691,7 +692,7 @@ class Core:
if len(data) == 0: if len(data) == 0:
logger.error('Will not insert empty block') logger.error('Will not insert empty block')
else: else:
addedHash = self.setData(metadata + data) addedHash = self.setData(metadata + b'\n' + data)
self.addToBlockDB(addedHash, selfInsert=True) self.addToBlockDB(addedHash, selfInsert=True)
self.setBlockType(addedHash, header) self.setBlockType(addedHash, header)
retData = addedHash retData = addedHash

9
onionr/onionr.py
View File

@ -25,7 +25,7 @@ import sys
if sys.version_info[0] == 2 or sys.version_info[1] < 5: if sys.version_info[0] == 2 or sys.version_info[1] < 5:
print('Error, Onionr requires Python 3.4+') print('Error, Onionr requires Python 3.4+')
sys.exit(1) sys.exit(1)
import os, base64, random, getpass, shutil, subprocess, requests, time, platform, datetime, re, json import os, base64, random, getpass, shutil, subprocess, requests, time, platform, datetime, re, json, getpass
from threading import Thread from threading import Thread
import api, core, config, logger, onionrplugins as plugins, onionrevents as events import api, core, config, logger, onionrplugins as plugins, onionrevents as events
import onionrutils import onionrutils
@ -108,7 +108,7 @@ class Onionr:
if not os.path.exists('data/blocks/'): if not os.path.exists('data/blocks/'):
os.mkdir('data/blocks/') os.mkdir('data/blocks/')
# Copy default plugins into plugins folder # Copy default plugins into plugins folder
if not os.path.exists(plugins.get_plugins_folder()): if not os.path.exists(plugins.get_plugins_folder()):
if os.path.exists('static-data/default-plugins/'): if os.path.exists('static-data/default-plugins/'):
names = [f for f in os.listdir("static-data/default-plugins/") if not os.path.isfile(f)] names = [f for f in os.listdir("static-data/default-plugins/") if not os.path.isfile(f)]
@ -463,7 +463,10 @@ class Onionr:
os.makedirs(plugins.get_plugins_folder(plugin_name)) os.makedirs(plugins.get_plugins_folder(plugin_name))
with open(plugins.get_plugins_folder(plugin_name) + '/main.py', 'a') as main: with open(plugins.get_plugins_folder(plugin_name) + '/main.py', 'a') as main:
main.write(open('static-data/default_plugin.py').read().replace('$user', os.getlogin()).replace('$date', datetime.datetime.now().strftime('%Y-%m-%d'))) main.write(open('static-data/default_plugin.py').read().replace('$user', os.getlogin()).replace('$date', datetime.datetime.now().strftime('%Y-%m-%d')).replace('$name', plugin_name))
with open(plugins.get_plugins_folder(plugin_name) + '/info.json', 'a') as main:
main.write(json.dumps({'author' : 'anonymous', 'description' : 'the default description of the plugin', 'version' : '1.0'}))
logger.info('Enabling plugin "%s"...' % plugin_name) logger.info('Enabling plugin "%s"...' % plugin_name)
plugins.enable(plugin_name, self) plugins.enable(plugin_name, self)

47
onionr/onionrcrypto.py
View File

@ -17,7 +17,7 @@
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
import nacl.signing, nacl.encoding, nacl.public, nacl.hash, nacl.secret, os, binascii, base64, hashlib, logger, onionrproofs, time import nacl.signing, nacl.encoding, nacl.public, nacl.hash, nacl.secret, os, binascii, base64, hashlib, logger, onionrproofs, time, math
class OnionrCrypto: class OnionrCrypto:
def __init__(self, coreInstance): def __init__(self, coreInstance):
@ -26,7 +26,7 @@ class OnionrCrypto:
self.keyPowFile = 'data/keyPow.txt' self.keyPowFile = 'data/keyPow.txt'
self.pubKey = None self.pubKey = None
self.privKey = None self.privKey = None
self.pubKeyPowToken = None self.pubKeyPowToken = None
self.pubKeyPowHash = None self.pubKeyPowHash = None
@ -88,7 +88,7 @@ class OnionrCrypto:
except nacl.exceptions.BadSignatureError: except nacl.exceptions.BadSignatureError:
pass pass
return retData return retData
def edSign(self, data, key, encodeResult=False): def edSign(self, data, key, encodeResult=False):
'''Ed25519 sign data''' '''Ed25519 sign data'''
try: try:
@ -196,7 +196,7 @@ class OnionrCrypto:
if returnEncoded: if returnEncoded:
decrypted = base64.b64encode(decrypted) decrypted = base64.b64encode(decrypted)
return decrypted return decrypted
def generateSymmetricPeer(self, peer): def generateSymmetricPeer(self, peer):
'''Generate symmetric key for a peer and save it to the peer database''' '''Generate symmetric key for a peer and save it to the peer database'''
key = self.generateSymmetric() key = self.generateSymmetric()
@ -212,7 +212,7 @@ class OnionrCrypto:
private_key = nacl.signing.SigningKey.generate() private_key = nacl.signing.SigningKey.generate()
public_key = private_key.verify_key.encode(encoder=nacl.encoding.Base32Encoder()) public_key = private_key.verify_key.encode(encoder=nacl.encoding.Base32Encoder())
return (public_key.decode(), private_key.encode(encoder=nacl.encoding.Base32Encoder()).decode()) return (public_key.decode(), private_key.encode(encoder=nacl.encoding.Base32Encoder()).decode())
def pubKeyHashID(self, pubkey=''): def pubKeyHashID(self, pubkey=''):
'''Accept a ed25519 public key, return a truncated result of X many sha3_256 hash rounds''' '''Accept a ed25519 public key, return a truncated result of X many sha3_256 hash rounds'''
if pubkey == '': if pubkey == '':
@ -234,10 +234,43 @@ class OnionrCrypto:
hasher = hashlib.sha3_256() hasher = hashlib.sha3_256()
hasher.update(data) hasher.update(data)
return hasher.hexdigest() return hasher.hexdigest()
def blake2bHash(self, data): def blake2bHash(self, data):
try: try:
data = data.encode() data = data.encode()
except AttributeError: except AttributeError:
pass pass
return nacl.hash.blake2b(data) return nacl.hash.blake2b(data)
def verifyPow(self, blockContent, metadata):
'''
Verifies the proof of work associated with a block
'''
retData = False
if not (('powToken' in metadata) and ('powHash' in metadata)):
return False
dataLen = len(blockContent)
expectedHash = self.blake2bHash(base64.b64decode(metadata['powToken']) + self.blake2bHash(blockContent.encode()))
difficulty = 0
try:
expectedHash = expectedHash.decode()
except AttributeError:
pass
if metadata['powHash'] == expectedHash:
difficulty = math.floor(dataLen / 1000000)
mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode()
puzzle = mainHash[:difficulty]
if metadata['powHash'][:difficulty] == puzzle:
# logger.debug('Validated block pow')
retData = True
else:
logger.debug("Invalid token (#1)")
else:
logger.debug('Invalid token (#2): Expected hash %s, got hash %s...' % (metadata['powHash'], expectedHash))
return retData

23
onionr/onionrpluginapi.py
View File

@ -18,7 +18,7 @@
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
import onionrplugins as plugins, logger import onionrplugins, logger
class DaemonAPI: class DaemonAPI:
def __init__(self, pluginapi): def __init__(self, pluginapi):
@ -52,34 +52,34 @@ class PluginAPI:
self.pluginapi = pluginapi self.pluginapi = pluginapi
def start(self, name): def start(self, name):
plugins.start(name) onionrplugins.start(name)
def stop(self, name): def stop(self, name):
plugins.stop(name) onionrplugins.stop(name)
def reload(self, name): def reload(self, name):
plugins.reload(name) onionrplugins.reload(name)
def enable(self, name): def enable(self, name):
plugins.enable(name) onionrplugins.enable(name)
def disable(self, name): def disable(self, name):
plugins.disable(name) onionrplugins.disable(name)
def event(self, name, data = {}): def event(self, name, data = {}):
events.event(name, data = data, onionr = self.pluginapi.get_onionr()) events.event(name, data = data, onionr = self.pluginapi.get_onionr())
def is_enabled(self, name): def is_enabled(self, name):
return plugins.is_enabled(name) return onionrplugins.is_enabled(name)
def get_enabled_plugins(self): def get_enabled_plugins(self):
return plugins.get_enabled() return onionrplugins.get_enabled()
def get_folder(self, name = None, absolute = True): def get_folder(self, name = None, absolute = True):
return plugins.get_plugins_folder(name = name, absolute = absolute) return onionrplugins.get_plugins_folder(name = name, absolute = absolute)
def get_data_folder(self, name, absolute = True): def get_data_folder(self, name, absolute = True):
return plugins.get_plugin_data_folder(name, absolute = absolute) return onionrplugins.get_plugin_data_folder(name, absolute = absolute)
def daemon_event(self, event, plugin = None): def daemon_event(self, event, plugin = None):
return # later make local command like /client/?action=makeEvent&event=eventname&module=modulename return # later make local command like /client/?action=makeEvent&event=eventname&module=modulename
@ -153,6 +153,9 @@ class pluginapi:
def get_utils(self): def get_utils(self):
return self.get_onionr().onionrUtils return self.get_onionr().onionrUtils
def get_crypto(self):
return self.get_core()._crypto
def get_daemonapi(self): def get_daemonapi(self):
return self.daemon return self.daemon

19
onionr/onionrplugins.py
View File

@ -62,17 +62,20 @@ def enable(name, onionr = None, start_event = True):
if exists(name): if exists(name):
enabled_plugins = get_enabled_plugins() enabled_plugins = get_enabled_plugins()
enabled_plugins.append(name) if not name in enabled_plugins:
config_plugins = config.get('plugins') enabled_plugins.append(name)
config_plugins['enabled'] = enabled_plugins config_plugins = config.get('plugins')
config.set('plugins', config_plugins, True) config_plugins['enabled'] = enabled_plugins
config.set('plugins', config_plugins, True)
events.call(get_plugin(name), 'enable', onionr) events.call(get_plugin(name), 'enable', onionr)
if start_event is True: if start_event is True:
start(name) start(name)
return True return True
else:
return False
else: else:
logger.error('Failed to enable plugin \"' + name + '\", disabling plugin.') logger.error('Failed to enable plugin \"' + name + '\", disabling plugin.')
disable(name) disable(name)

33
onionr/onionrutils.py
View File

@ -71,7 +71,7 @@ class OnionrUtils:
if block == '': if block == '':
logger.error('Could not send PM') logger.error('Could not send PM')
else: else:
logger.info('Sent PM, hash: ' + block) logger.info('Sent PM, hash: %s' % block)
except Exception as error: except Exception as error:
logger.error('Failed to send PM.', error=error) logger.error('Failed to send PM.', error=error)
@ -103,14 +103,14 @@ class OnionrUtils:
for key in newKeyList.split(','): for key in newKeyList.split(','):
key = key.split('-') key = key.split('-')
if len(key[0]) > 60 or len(key[1]) > 1000: if len(key[0]) > 60 or len(key[1]) > 1000:
logger.warn(key[0] + ' or its pow value is too large.') logger.warn('%s or its pow value is too large.' % key[0])
continue continue
if self._core._crypto.blake2bHash(base64.b64decode(key[1]) + key[0].encode()).startswith('0000'): if self._core._crypto.blake2bHash(base64.b64decode(key[1]) + key[0].encode()).startswith('0000'):
if not key[0] in self._core.listPeers(randomOrder=False) and type(key) != None and key[0] != self._core._crypto.pubKey: if not key[0] in self._core.listPeers(randomOrder=False) and type(key) != None and key[0] != self._core._crypto.pubKey:
if self._core.addPeer(key[0], key[1]): if self._core.addPeer(key[0], key[1]):
retVal = True retVal = True
else: else:
logger.warn(key[0] + 'pow failed') logger.warn('%s pow failed' % key[0])
return retVal return retVal
except Exception as error: except Exception as error:
logger.error('Failed to merge keys.', error=error) logger.error('Failed to merge keys.', error=error)
@ -127,7 +127,7 @@ class OnionrUtils:
for adder in newAdderList.split(','): for adder in newAdderList.split(','):
if not adder in self._core.listAdders(randomOrder = False) and adder.strip() != self.getMyAddress(): if not adder in self._core.listAdders(randomOrder = False) and adder.strip() != self.getMyAddress():
if self._core.addAddress(adder): if self._core.addAddress(adder):
logger.info('Added ' + adder + ' to db.', timestamp = True) logger.info('Added %s to db.' % adder, timestamp = True)
retVal = True retVal = True
else: else:
logger.debug('%s is either our address or already in our DB' % adder) logger.debug('%s is either our address or already in our DB' % adder)
@ -156,7 +156,7 @@ class OnionrUtils:
retData = requests.get('http://' + open('data/host.txt', 'r').read() + ':' + str(config.get('client')['port']) + '/client/?action=' + command + '&token=' + str(config.get('client')['client_hmac']) + '&timingToken=' + self.timingToken).text retData = requests.get('http://' + open('data/host.txt', 'r').read() + ':' + str(config.get('client')['port']) + '/client/?action=' + command + '&token=' + str(config.get('client')['client_hmac']) + '&timingToken=' + self.timingToken).text
except Exception as error: except Exception as error:
if not silent: if not silent:
logger.error('Failed to make local request (command: ' + str(command) + ').', error=error) logger.error('Failed to make local request (command: %s).' % command, error=error)
retData = False retData = False
return retData return retData
@ -334,7 +334,7 @@ class OnionrUtils:
''' '''
Find, decrypt, and return array of PMs (array of dictionary, {from, text}) Find, decrypt, and return array of PMs (array of dictionary, {from, text})
''' '''
#blocks = self._core.getBlockList().split('\n') #blocks = self._core.getBlockList()
blocks = self._core.getBlocksByType('pm') blocks = self._core.getBlocksByType('pm')
message = '' message = ''
sender = '' sender = ''
@ -344,8 +344,8 @@ class OnionrUtils:
try: try:
with open('data/blocks/' + i + '.dat', 'r') as potentialMessage: with open('data/blocks/' + i + '.dat', 'r') as potentialMessage:
potentialMessage = potentialMessage.read() potentialMessage = potentialMessage.read()
blockMetadata = json.loads(potentialMessage[:potentialMessage.rfind('}') + 1]) blockMetadata = json.loads(potentialMessage[:potentialMessage.find('\n')])
blockContent = potentialMessage[potentialMessage.rfind('}') + 1:] blockContent = potentialMessage[potentialMessage.find('\n') + 1:]
try: try:
message = self._core._crypto.pubKeyDecrypt(blockContent, encodedData=True, anonymous=True) message = self._core._crypto.pubKeyDecrypt(blockContent, encodedData=True, anonymous=True)
@ -362,8 +362,7 @@ class OnionrUtils:
except json.decoder.JSONDecodeError: except json.decoder.JSONDecodeError:
pass pass
else: else:
print('--------------------') logger.info('Decrypted %s:' % i)
logger.info('Decrypted ' + i + ':')
logger.info(message["msg"]) logger.info(message["msg"])
signer = message["id"] signer = message["id"]
@ -371,16 +370,16 @@ class OnionrUtils:
if self.validatePubKey(signer): if self.validatePubKey(signer):
if self._core._crypto.edVerify(message["msg"], signer, sig, encodedData=True): if self._core._crypto.edVerify(message["msg"], signer, sig, encodedData=True):
logger.info("Good signature by " + signer) logger.info("Good signature by %s" % signer)
else: else:
logger.warn("Bad signature by " + signer) logger.warn("Bad signature by %s" % signer)
else: else:
logger.warn("Bad sender id: " + signer) logger.warn('Bad sender id: %s' % signer)
except FileNotFoundError: except FileNotFoundError:
pass pass
except Exception as error: except Exception as error:
logger.error('Failed to open block ' + str(i) + '.', error=error) logger.error('Failed to open block %s.' % i, error=error)
return return
def getPeerByHashId(self, hash): def getPeerByHashId(self, hash):
@ -438,14 +437,14 @@ class OnionrUtils:
scanDir += '/' scanDir += '/'
for block in glob.glob(scanDir + "*.dat"): for block in glob.glob(scanDir + "*.dat"):
if block.replace(scanDir, '').replace('.dat', '') not in blockList: if block.replace(scanDir, '').replace('.dat', '') not in blockList:
logger.info("Found new block on dist " + block) logger.info('Found new block on dist %s' % block)
with open(block, 'rb') as newBlock: with open(block, 'rb') as newBlock:
block = block.replace(scanDir, '').replace('.dat', '') block = block.replace(scanDir, '').replace('.dat', '')
if self._core._crypto.sha3Hash(newBlock.read()) == block.replace('.dat', ''): if self._core._crypto.sha3Hash(newBlock.read()) == block.replace('.dat', ''):
self._core.addToBlockDB(block.replace('.dat', ''), dataSaved=True) self._core.addToBlockDB(block.replace('.dat', ''), dataSaved=True)
logger.info('Imported block.') logger.info('Imported block %s.' % block)
else: else:
logger.warn('Failed to verify hash for ' + block) logger.warn('Failed to verify hash for %s' % block)
def progressBar(self, value = 0, endvalue = 100, width = None): def progressBar(self, value = 0, endvalue = 100, width = None):

5
onionr/static-data/default-plugins/gui/info.json Normal file
View File

@ -0,0 +1,5 @@
{
"name" : "gui",
"version" : "1.0",
"author" : "onionr"
}

4
onionr/static-data/default-plugins/gui/main.py
View File

@ -19,6 +19,8 @@
import logger, config import logger, config
import os, sqlite3, core import os, sqlite3, core
plugin_name = 'gui'
def sendMessage(): def sendMessage():
global sendEntry global sendEntry
@ -73,7 +75,7 @@ def openGUI():
nodeInfo = tkinter.Frame(root) nodeInfo = tkinter.Frame(root)
keyInfo = tkinter.Frame(root) keyInfo = tkinter.Frame(root)
hostname = pluginapi.get_onionr().get_hostname() hostname = pluginapi.get_onionr().get_hostname()
logger.debug('hostname: %s' % hostname) logger.debug('hostname: %s' % hostname)
idText = hostname idText = hostname

5
onionr/static-data/default-plugins/pluginmanager/info.json Normal file
View File

@ -0,0 +1,5 @@
{
"name" : "pluginmanager",
"version" : "1.0",
"author" : "onionr"
}

412
onionr/static-data/default-plugins/pluginmanager/main.py
View File

@ -4,11 +4,11 @@
# useful libraries # useful libraries
import logger, config import logger, config
import os, sys, json, time, random import os, sys, json, time, random, shutil, base64, getpass, datetime, re
plugin_name = 'pluginmanager' plugin_name = 'pluginmanager'
keys_data = {'keys' : {}} keys_data = {'keys' : {}, 'plugins' : [], 'repositories' : {}}
# key functions # key functions
@ -35,6 +35,7 @@ def getKey(plugin):
Returns the public key for a given plugin Returns the public key for a given plugin
''' '''
global keys_data
readKeys() readKeys()
return (keys_data['keys'][plugin] if plugin in keys_data['keys'] else None) return (keys_data['keys'][plugin] if plugin in keys_data['keys'] else None)
@ -43,9 +44,72 @@ def saveKey(plugin, key):
Saves the public key for a plugin to keystore Saves the public key for a plugin to keystore
''' '''
global keys_data
readKeys()
keys_data['keys'][plugin] = key keys_data['keys'][plugin] = key
writeKeys() writeKeys()
def getPlugins():
'''
Returns a list of plugins installed by the plugin manager
'''
global keys_data
readKeys()
return keys_data['plugins']
def addPlugin(plugin):
'''
Saves the plugin name, to remember that it was installed by the pluginmanager
'''
global keys_data
readKeys()
if not plugin in keys_data['plugins']:
keys_data['plugins'].append(plugin)
writeKeys()
def removePlugin(plugin):
'''
Removes the plugin name from the pluginmanager's records
'''
global keys_data
readKeys()
if plugin in keys_data['plugins']:
keys_data['plugins'].remove(plugin)
writeKeys()
def getRepositories():
'''
Returns a list of plugins installed by the plugin manager
'''
global keys_data
readKeys()
return keys_data['repositories']
def addRepository(repositories, data):
'''
Saves the plugin name, to remember that it was installed by the pluginmanager
'''
global keys_data
readKeys()
keys_data['repositories'][repositories] = data
writeKeys()
def removeRepository(repositories):
'''
Removes the plugin name from the pluginmanager's records
'''
global keys_data
readKeys()
if plugin in keys_data['repositories']:
del keys_data['repositories'][repositories]
writeKeys()
def check(): def check():
''' '''
Checks to make sure the keystore file still exists Checks to make sure the keystore file still exists
@ -56,19 +120,220 @@ def check():
if not os.path.isfile(keys_file): if not os.path.isfile(keys_file):
writeKeys() writeKeys()
# plugin management
def sanitize(name):
return re.sub('[^0-9a-zA-Z]+', '', str(name).lower())[:255]
def blockToPlugin(block):
try:
blockContent = pluginapi.get_core().getData(block)
blockContent = blockContent[blockContent.rfind(b'\n') + 1:].decode()
blockContent = json.loads(blockContent)
name = sanitize(blockContent['name'])
author = blockContent['author']
date = blockContent['date']
version = None
if 'version' in blockContent['info']:
version = blockContent['info']['version']
content = base64.b64decode(blockContent['content'].encode())
source = pluginapi.plugins.get_data_folder(plugin_name) + 'plugin.zip'
destination = pluginapi.plugins.get_folder(name)
with open(source, 'wb') as f:
f.write(content)
if os.path.exists(destination) and not os.path.isfile(destination):
shutil.rmtree(destination)
shutil.unpack_archive(source, destination)
pluginapi.plugins.enable(name)
logger.info('Installation of %s complete.' % name)
return True
except Exception as e:
logger.error('Failed to install plugin.', error = e, timestamp = False)
return False
def pluginToBlock(plugin, import_block = True):
try:
plugin = sanitize(plugin)
directory = pluginapi.get_pluginapi().get_folder(plugin)
data_directory = pluginapi.get_pluginapi().get_data_folder(plugin)
zipfile = pluginapi.get_pluginapi().get_data_folder(plugin_name) + 'plugin.zip'
if os.path.exists(directory) and not os.path.isfile(directory):
if os.path.exists(data_directory) and not os.path.isfile(data_directory):
shutil.rmtree(data_directory)
if os.path.exists(zipfile) and os.path.isfile(zipfile):
os.remove(zipfile)
if os.path.exists(directory + '__pycache__') and not os.path.isfile(directory + '__pycache__'):
shutil.rmtree(directory + '__pycache__')
shutil.make_archive(zipfile[:-4], 'zip', directory)
data = base64.b64encode(open(zipfile, 'rb').read())
author = getpass.getuser()
description = 'Default plugin description'
info = {"name" : plugin}
try:
if os.path.exists(directory + 'info.json'):
info = json.loads(open(directory + 'info.json').read())
if 'author' in info:
author = info['author']
if 'description' in info:
description = info['description']
except:
pass
metadata = {'author' : author, 'date' : str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')), 'name' : plugin, 'info' : info, 'compiled-by' : plugin_name, 'content' : data.decode('utf-8'), 'description' : description}
hash = pluginapi.get_core().insertBlock(json.dumps(metadata), header = 'plugin', sign = True)
if import_block:
pluginapi.get_utils().importNewBlocks()
return hash
else:
logger.error('Plugin %s does not exist.' % plugin)
except Exception as e:
logger.error('Failed to convert plugin to block.', error = e, timestamp = False)
return False
def parseBlock(hash, key):# deal with block metadata
blockContent = pluginapi.get_core().getData(hash)
try:
blockMetadata = json.loads(blockContent[:blockContent.decode().find('\n')].decode())
try:
blockMeta2 = json.loads(blockMetadata['meta'])
except KeyError:
blockMeta2 = {'type': ''}
pass
blockContent = blockContent[blockContent.rfind(b'\n') + 1:]
try:
blockContent = blockContent.decode()
except AttributeError:
pass
if not pluginapi.get_crypto().verifyPow(blockContent, blockMeta2):
logger.debug("(pluginmanager): %s has invalid or insufficient proof of work" % str(hash))
return False
if not (('sig' in blockMetadata)): # and ('id' in blockMeta2)
logger.debug('(pluginmanager): %s is missing required parameters' % hash)
return False
else:
if pluginapi.get_crypto().edVerify(blockMetadata['meta'] + '\n' + blockContent, key, blockMetadata['sig'], encodedData=True):
# logger.debug('(pluginmanager): %s was signed' % str(hash))
return True
else:
# logger.debug('(pluginmanager): %s has an invalid signature' % str(hash))
return False
except json.decoder.JSONDecodeError as e:
logger.error('(pluginmanager): Could not decode block metadata.', error = e, timestamp = False)
return False
def installBlock(block):
try:
blockContent = pluginapi.get_core().getData(block)
blockContent = blockContent[blockContent.rfind(b'\n') + 1:].decode()
blockContent = json.loads(blockContent)
name = sanitize(blockContent['name'])
author = blockContent['author']
date = blockContent['date']
version = None
if 'version' in blockContent['info']:
version = blockContent['info']['version']
install = False
logger.info(('Will install %s' + (' v' + version if not version is None else '') + ' (%s), by %s') % (name, date, author))
# TODO: Convert to single line if statement
if os.path.exists(pluginapi.plugins.get_folder(name)):
install = logger.confirm(message = 'Continue with installation (will overwrite existing plugin) %s?')
else:
install = logger.confirm(message = 'Continue with installation %s?')
if install:
blockToPlugin(block)
addPlugin(name)
else:
logger.info('Installation cancelled.')
return False
return True
except Exception as e:
logger.error('Failed to install plugin.', error = e, timestamp = False)
return False
def uninstallPlugin(plugin):
try:
plugin = sanitize(plugin)
pluginFolder = pluginapi.plugins.get_folder(plugin)
exists = (os.path.exists(pluginFolder) and not os.path.isfile(pluginFolder))
installedByPluginManager = plugin in getPlugins()
remove = False
if not exists:
logger.warn('Plugin %s does not exist.' % plugin, timestamp = False)
return False
default = 'y'
if not installedByPluginManager:
logger.warn('The plugin %s was not installed by %s.' % (plugin, plugin_name), timestamp = False)
default = 'n'
remove = logger.confirm(message = 'All plugin data will be lost. Are you sure you want to proceed %s?', default = default)
if remove:
if installedByPluginManager:
removePlugin(plugin)
pluginapi.plugins.disable(plugin)
shutil.rmtree(pluginFolder)
logger.info('Uninstallation of %s complete.' % plugin)
return True
else:
logger.info('Uninstallation cancelled.')
except Exception as e:
logger.error('Failed to uninstall plugin.', error = e)
return False
# command handlers # command handlers
def help(): def help():
logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin> [public key/block hash]') logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin> [public key/block hash]')
logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin> [public key/block hash]')
def commandInstallPlugin(): def commandInstallPlugin():
logger.warn('This feature is not functional or is still in development.')
if len(sys.argv) >= 3: if len(sys.argv) >= 3:
check() check()
pluginname = sys.argv[2] pluginname = sys.argv[2]
pkobh = None # public key or block hash pkobh = None # public key or block hash
version = None
if ':' in pluginname:
details = pluginname
pluginname = sanitize(details[0])
version = details[1]
sanitize(pluginname)
if len(sys.argv) >= 4: if len(sys.argv) >= 4:
# public key or block hash specified # public key or block hash specified
pkobh = sys.argv[3] pkobh = sys.argv[3]
@ -77,7 +342,54 @@ def commandInstallPlugin():
pkobh = getKey(pluginname) pkobh = getKey(pluginname)
if pkobh is None: if pkobh is None:
logger.error('No key for this plugin found in keystore, please specify.') # still nothing found, try searching repositories
logger.info('Searching for public key in repositories...')
try:
repos = getRepositories()
distributors = list()
for repo, records in repos.items():
if pluginname in records:
logger.debug('Found %s in repository %s for plugin %s.' % (records[pluginname], repo, pluginname))
distributors.append(records[pluginname])
if len(distributors) != 0:
distributor = None
if len(distributors) == 1:
logger.info('Found distributor: %s' % distributors[0])
distributor = distributors[0]
else:
distributors_message = ''
index = 1
for dist in distributors:
distributors_message += ' ' + logger.colors.bold + str(index) + ') ' + logger.colors.reset + str(dist) + '\n'
index += 1
logger.info((logger.colors.bold + 'Found distributors (%s):' + logger.colors.reset + '\n' + distributors_message) % len(distributors))
valid = False
while not valid:
choice = logger.readline('Select the number of the key to use, from 1 to %s, or press Ctrl+C to cancel:' % (index - 1))
try:
if int(choice) < index and int(choice) >= 1:
distributor = distributors[int(choice)]
valid = True
except KeyboardInterrupt:
logger.info('Installation cancelled.')
return True
except:
pass
if not distributor is None:
pkobh = distributor
except Exception as e:
logger.warn('Failed to lookup plugin in repositories.', timestamp = False)
logger.error('asdf', error = e, timestamp = False)
if pkobh is None:
logger.error('No key for this plugin found in keystore or repositories, please specify.')
help() help()
return True return True
@ -96,18 +408,13 @@ def commandInstallPlugin():
if valid_hash and not real_block: if valid_hash and not real_block:
logger.error('Block hash not found. Perhaps it has not been synced yet?') logger.error('Block hash not found. Perhaps it has not been synced yet?')
logger.debug('Is valid hash, but does not belong to a known block.') logger.debug('Is valid hash, but does not belong to a known block.')
return True return True
elif valid_hash and real_block: elif valid_hash and real_block:
blockhash = str(pkobh) blockhash = str(pkobh)
logger.debug('Using block %s...' % blockhash) logger.debug('Using block %s...' % blockhash)
logger.info('Downloading plugin...') installBlock(blockhash)
for i in range(0, 100):
pluginapi.get_utils().progressBar(i, 100)
time.sleep(random.random() / 5)
logger.info('Finished downloading plugin, verifying and installing...')
time.sleep(1)
logger.info('Installation successful.')
elif valid_key and not real_key: elif valid_key and not real_key:
logger.error('Public key not found. Try adding the node by address manually, if possible.') logger.error('Public key not found. Try adding the node by address manually, if possible.')
logger.debug('Is valid key, but the key is not a known one.') logger.debug('Is valid key, but the key is not a known one.')
@ -117,28 +424,86 @@ def commandInstallPlugin():
saveKey(pluginname, pkobh) saveKey(pluginname, pkobh)
logger.info('Downloading plugin...') blocks = pluginapi.get_core().getBlocksByType('plugin')
for i in range(0, 100):
pluginapi.get_utils().progressBar(i, 100) signedBlocks = list()
time.sleep(random.random() / 5)
logger.info('Finished downloading plugin, verifying and installing...') for hash in blocks:
time.sleep(1) if parseBlock(hash, publickey):
logger.info('Installation successful.') signedBlocks.append(hash)
mostRecentTimestamp = None
mostRecentVersionBlock = None
for hash in signedBlocks:
try:
blockContent = pluginapi.get_core().getData(hash)
blockContent = blockContent[blockContent.rfind(b'\n') + 1:].decode()
blockContent = json.loads(blockContent)
if not (('author' in blockContent) and ('info' in blockContent) and ('date' in blockContent) and ('name' in blockContent)):
raise ValueError('Missing required parameter `date` in block %s.' % hash)
blockDatetime = datetime.datetime.strptime(blockContent['date'], '%Y-%m-%d %H:%M:%S')
if blockContent['name'] == pluginname:
if ('version' in blockContent['info']) and (blockContent['info']['version'] == version) and (not version is None):
mostRecentTimestamp = blockDatetime
mostRecentVersionBlock = hash
break
elif mostRecentTimestamp is None:
mostRecentTimestamp = blockDatetime
mostRecentVersionBlock = hash
elif blockDatetime > mostRecentTimestamp:
mostRecentTimestamp = blockDatetime
mostRecentVersionBlock = hash
except Exception as e:
pass
logger.warn('Only continue the installation is you are absolutely certain that you trust the plugin distributor. Public key of plugin distributor: %s' % publickey, timestamp = False)
installBlock(mostRecentVersionBlock)
else: else:
logger.error('Unknown data "%s"; must be public key or block hash.' % str(pkobh)) logger.error('Unknown data "%s"; must be public key or block hash.' % str(pkobh))
return return
else: else:
help() logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin> [public key/block hash]')
return True return True
def commandUninstallPlugin(): def commandUninstallPlugin():
logger.info('This feature has not been created yet. Please check back later.') if len(sys.argv) >= 3:
return uninstallPlugin(sys.argv[2])
else:
logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin>')
return True
def commandSearchPlugin(): def commandSearchPlugin():
logger.info('This feature has not been created yet. Please check back later.') logger.info('This feature has not been created yet. Please check back later.')
return return True
def commandAddRepository():
logger.info('This feature has not been created yet. Please check back later.')
return True
def commandRemoveRepository():
logger.info('This feature has not been created yet. Please check back later.')
return True
def commandPublishPlugin():
if len(sys.argv) >= 3:
check()
pluginname = sanitize(sys.argv[2])
pluginfolder = pluginapi.plugins.get_folder(pluginname)
if os.path.exists(pluginfolder) and not os.path.isfile(pluginfolder):
block = pluginToBlock(pluginname)
logger.info('Plugin saved in block %s.' % block)
else:
logger.error('Plugin %s does not exist.' % pluginname, timestamp = False)
else:
logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin>')
# event listeners # event listeners
@ -151,6 +516,9 @@ def on_init(api, data = None):
api.commands.register(['install-plugin', 'installplugin', 'plugin-install', 'install', 'plugininstall'], commandInstallPlugin) api.commands.register(['install-plugin', 'installplugin', 'plugin-install', 'install', 'plugininstall'], commandInstallPlugin)
api.commands.register(['remove-plugin', 'removeplugin', 'plugin-remove', 'uninstall-plugin', 'uninstallplugin', 'plugin-uninstall', 'uninstall', 'remove', 'pluginremove'], commandUninstallPlugin) api.commands.register(['remove-plugin', 'removeplugin', 'plugin-remove', 'uninstall-plugin', 'uninstallplugin', 'plugin-uninstall', 'uninstall', 'remove', 'pluginremove'], commandUninstallPlugin)
api.commands.register(['search', 'filter-plugins', 'search-plugins', 'searchplugins', 'search-plugin', 'searchplugin', 'findplugin', 'find-plugin', 'filterplugin', 'plugin-search', 'pluginsearch'], commandSearchPlugin) api.commands.register(['search', 'filter-plugins', 'search-plugins', 'searchplugins', 'search-plugin', 'searchplugin', 'findplugin', 'find-plugin', 'filterplugin', 'plugin-search', 'pluginsearch'], commandSearchPlugin)
api.commands.register(['add-repo', 'add-repository', 'addrepo', 'addrepository', 'repository-add', 'repo-add', 'repoadd', 'addrepository', 'add-plugin-repository', 'add-plugin-repo', 'add-pluginrepo', 'add-pluginrepository', 'addpluginrepo', 'addpluginrepository'], commandAddRepository)
api.commands.register(['remove-repo', 'remove-repository', 'removerepo', 'removerepository', 'repository-remove', 'repo-remove', 'reporemove', 'removerepository', 'remove-plugin-repository', 'remove-plugin-repo', 'remove-pluginrepo', 'remove-pluginrepository', 'removepluginrepo', 'removepluginrepository', 'rm-repo', 'rm-repository', 'rmrepo', 'rmrepository', 'repository-rm', 'repo-rm', 'reporm', 'rmrepository', 'rm-plugin-repository', 'rm-plugin-repo', 'rm-pluginrepo', 'rm-pluginrepository', 'rmpluginrepo', 'rmpluginrepository'], commandRemoveRepository)
api.commands.register(['publish-plugin', 'plugin-publish', 'publishplugin', 'pluginpublish', 'publish'], commandPublishPlugin)
# add help menus once the features are actually implemented # add help menus once the features are actually implemented

4
onionr/static-data/default_plugin.py
View File

@ -1,11 +1,13 @@
''' '''
Default plugin template file $name plugin template file.
Generated on $date by $user. Generated on $date by $user.
''' '''
# Imports some useful libraries # Imports some useful libraries
import logger, config import logger, config
plugin_name = '$name'
def on_init(api, data = None): def on_init(api, data = None):
''' '''
This event is called after Onionr is initialized, but before the command This event is called after Onionr is initialized, but before the command