kev/Onionr
1
0
Fork 0
Code Issues 24 Pull Requests Projects Releases 2 Wiki Activity

more progress in removing onionrutils class

Browse Source
This commit is contained in:
Kevin Froman 2019-06-25 03:21:36 -05:00
parent d378340099
commit 909c002dc4
37 changed files with 164 additions and 189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
onionr/api.py
View File

@ -23,8 +23,7 @@ from gevent import Timeout
import flask
from flask import request, Response, abort, send_from_directory
import core
from onionrblockapi import Block
import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config
import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config, onionrblockapi
import httpapi
from httpapi import friendsapi, profilesapi, configapi, miscpublicapi
from onionrservices import httpheaders
@ -337,7 +336,7 @@ class API:
resp = ''
if self._core._utils.validateHash(name):
try:
resp = Block(name, decrypt=True).bcontent
resp = onionrblockapi.Block(name, decrypt=True).bcontent
except TypeError:
pass
else:
@ -374,7 +373,7 @@ class API:
resp = 'Not Found'
if self._core._utils.validateHash(bHash):
try:
resp = Block(bHash).bcontent
resp = onionrblockapi.Block(bHash).bcontent
except onionrexceptions.NoDataAvailable:
abort(404)
except TypeError:
@ -505,7 +504,7 @@ class API:
def getBlockData(self, bHash, decrypt=False, raw=False, headerOnly=False):
assert self._core._utils.validateHash(bHash)
bl = Block(bHash, core=self._core)
bl = onionrblockapi.Block(bHash, core=self._core)
if decrypt:
bl.decrypt()
if bl.isEncrypted and not bl.decrypted:
@ -521,8 +520,8 @@ class API:
pass
else:
validSig = False
signer = self._core._utils.bytesToStr(bl.signer)
if bl.isSigned() and self._core._utils.validatePubKey(signer) and bl.isSigner(signer):
signer = onionrutils.bytes_to_str(bl.signer)
if bl.isSigned() and onionrutils.stringvalidators.validate_pub_key(signer) and bl.isSigner(signer):
validSig = True
bl.bheader['validSig'] = validSig
bl.bheader['meta'] = ''

9
onionr/blockimporter.py
View File

@ -18,6 +18,7 @@
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import core, onionrexceptions, logger
from onionrutils import validatemetadata, blockmetadata
def importBlockFromData(content, coreInst):
retData = False
@ -34,17 +35,17 @@ def importBlockFromData(content, coreInst):
except AttributeError:
pass
metas = coreInst._utils.getBlockMetadataFromData(content) # returns tuple(metadata, meta), meta is also in metadata
metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
metadata = metas[0]
if coreInst._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid
if validatemetadata(metadata, metas[2]): # check if metadata is valid
if coreInst._crypto.verifyPow(content): # check if POW is enough/correct
logger.info('Block passed proof, saving.')
logger.info('Block passed proof, saving.', terminal=True)
try:
blockHash = coreInst.setData(content)
except onionrexceptions.DiskAllocationReached:
pass
else:
coreInst.addToBlockDB(blockHash, dataSaved=True)
coreInst._utils.processBlockMetadata(blockHash) # caches block metadata values to block database
blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database
retData = True
return retData

5
onionr/communicatorutils/downloadblocks.py
View File

@ -19,6 +19,7 @@
'''
import communicator, onionrexceptions
import logger, onionrpeers
from onionrutils import blockmetadata
def download_blocks_from_communicator(comm_inst):
assert isinstance(comm_inst, communicator.OnionrCommunicatorDaemon)
@ -72,7 +73,7 @@ def download_blocks_from_communicator(comm_inst):
pass
if realHash == blockHash:
content = content.decode() # decode here because sha3Hash needs bytes above
metas = comm_inst._core._utils.getBlockMetadataFromData(content) # returns tuple(metadata, meta), meta is also in metadata
metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
metadata = metas[0]
if comm_inst._core._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid, and verify nonce
if comm_inst._core._crypto.verifyPow(content): # check if POW is enough/correct
@ -84,7 +85,7 @@ def download_blocks_from_communicator(comm_inst):
removeFromQueue = False
else:
comm_inst._core.addToBlockDB(blockHash, dataSaved=True)
comm_inst._core._utils.processBlockMetadata(blockHash) # caches block metadata values to block database
blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database
else:
logger.warn('POW failed for block %s.' % blockHash)
else:

2
onionr/communicatorutils/servicecreator.py
View File

@ -32,7 +32,7 @@ def service_creator(daemon):
if not b in daemon.active_services:
bl = onionrblockapi.Block(b, core=core, decrypt=True)
bs = utils.bytesToStr(bl.bcontent) + '.onion'
if utils.validatePubKey(bl.signer) and stringvalidators.validate_transport(bs):
if stringvalidators.validate_pub_key(bl.signer) and stringvalidators.validate_transport(bs):
signer = utils.bytesToStr(bl.signer)
daemon.active_services.append(b)
daemon.active_services.append(signer)

15
onionr/core.py
View File

@ -28,7 +28,8 @@ from onionrusers import onionrusers
from onionrstorage import removeblock, setdata
import dbcreator, onionrstorage, serializeddata, subprocesspow
from etc import onionrvalues, powchoice
from onionrutils import localcommand
from onionrutils import localcommand, stringvalidators, bytesconverter, epoch
from onionrutils import blockmetadata
class Core:
def __init__(self, torPort=0):
@ -320,9 +321,9 @@ class Core:
if type(data) is None:
raise ValueError('Data cannot be none')
createTime = self._utils.getRoundedEpoch()
createTime = epoch.get_epoch()
dataNonce = self._utils.bytesToStr(self._crypto.sha3Hash(data))
dataNonce = bytesconverter.bytes_to_str(self._crypto.sha3Hash(data))
try:
with open(self.dataNonceFile, 'r') as nonces:
if dataNonce in nonces:
@ -395,7 +396,7 @@ class Core:
signature = self._crypto.symmetricEncrypt(signature, key=symKey, returnEncoded=True).decode()
signer = self._crypto.symmetricEncrypt(signer, key=symKey, returnEncoded=True).decode()
elif encryptType == 'asym':
if self._utils.validatePubKey(asymPeer):
if stringvalidators.validate_pub_key(asymPeer):
# Encrypt block data with forward secrecy key first, but not meta
jsonMeta = json.dumps(meta)
jsonMeta = self._crypto.pubKeyEncrypt(jsonMeta, asymPeer, encodedData=True).decode()
@ -438,13 +439,13 @@ class Core:
localcommand.local_command(self, '/waitforshare/' + retData, post=True, maxWait=5)
self.daemonQueueAdd('uploadBlock', retData)
self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
self._utils.processBlockMetadata(retData)
blockmetadata.process_block_metadata(retData)
if retData != False:
if plaintextPeer == onionrvalues.DENIABLE_PEER_ADDRESS:
events.event('insertdeniable', {'content': plaintext, 'meta': plaintextMeta, 'hash': retData, 'peer': self._utils.bytesToStr(asymPeer)}, onionr = self.onionrInst, threaded = True)
events.event('insertdeniable', {'content': plaintext, 'meta': plaintextMeta, 'hash': retData, 'peer': bytesconverter.bytes_to_str(asymPeer)}, onionr = self.onionrInst, threaded = True)
else:
events.event('insertblock', {'content': plaintext, 'meta': plaintextMeta, 'hash': retData, 'peer': self._utils.bytesToStr(asymPeer)}, onionr = self.onionrInst, threaded = True)
events.event('insertblock', {'content': plaintext, 'meta': plaintextMeta, 'hash': retData, 'peer': bytesconverter.bytes_to_str(asymPeer)}, onionr = self.onionrInst, threaded = True)
return retData
def introduceNode(self):

3
onionr/coredb/blockmetadb/add.py
View File

@ -1,4 +1,5 @@
import os, sqlite3
import onionrutils
def add_to_block_DB(core_inst, newHash, selfInsert=False, dataSaved=False):
'''
Add a hash value to the block db
@ -8,7 +9,7 @@ def add_to_block_DB(core_inst, newHash, selfInsert=False, dataSaved=False):
if not os.path.exists(core_inst.blockDB):
raise Exception('Block db does not exist')
if core_inst._utils.hasBlock(newHash):
if onionrutils.has_block(core_inst, newHash):
return
conn = sqlite3.connect(core_inst.blockDB, timeout=30)
c = conn.cursor()

2
onionr/coredb/keydb/addkeys.py
View File

@ -10,7 +10,7 @@ def add_peer(core_inst, peerID, name=''):
raise ValueError("specified id is already known")
# This function simply adds a peer to the DB
if not core_inst._utils.validatePubKey(peerID):
if not stringvalidators.validate_pub_key(peerID):
return False
events.event('pubkey_add', data = {'key': peerID}, onionr = core_inst.onionrInst)

8
onionr/httpapi/miscpublicapi/getblocks.py
View File

@ -18,12 +18,12 @@
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
from flask import Response, abort
import config
import config, onionrutils
def get_public_block_list(clientAPI, publicAPI, request):
# Provide a list of our blocks, with a date offset
dateAdjust = request.args.get('date')
bList = clientAPI._core.getBlockList(dateRec=dateAdjust)
if config.get('general.hide_created_blocks', True):
if clientAPI._core.config.get('general.hide_created_blocks', True):
for b in publicAPI.hideBlocks:
if b in bList:
# Don't share blocks we created if they haven't been *uploaded* yet, makes it harder to find who created a block
@ -34,14 +34,14 @@ def get_block_data(clientAPI, publicAPI, data):
'''data is the block hash in hex'''
resp = ''
if clientAPI._utils.validateHash(data):
if not config.get('general.hide_created_blocks', True) or data not in publicAPI.hideBlocks:
if not clientAPI._core.config.get('general.hide_created_blocks', True) or data not in publicAPI.hideBlocks:
if data in clientAPI._core.getBlockList():
block = clientAPI.getBlockData(data, raw=True)
try:
block = block.encode() # Encode in case data is binary
except AttributeError:
abort(404)
block = clientAPI._core._utils.strToBytes(block)
block = onionrutils.str_to_bytes(block)
resp = block
if len(resp) == 0:
abort(404)

1
onionr/netcontroller.py
View File

@ -20,7 +20,6 @@
import subprocess, os, sys, time, signal, base64, socket
from shutil import which
import logger, config
from onionrblockapi import Block
config.reload()
def getOpenPort():
# taken from (but modified) https://stackoverflow.com/a/2838309 by https://stackoverflow.com/users/133374/albert ccy-by-sa-3 https://creativecommons.org/licenses/by-sa/3.0/

11
onionr/onionr.py
View File

@ -21,6 +21,10 @@
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import sys
ONIONR_TAGLINE = 'Private P2P Communication - GPLv3 - https://Onionr.net'
ONIONR_VERSION = '0.0.0' # for debugging and stuff
ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION)
API_VERSION = '0' # increments of 1; only change when something fundamental about how the API works changes. This way other nodes know how to communicate without learning too much information about you.
MIN_PY_VERSION = 6
if sys.version_info[0] == 2 or sys.version_info[1] < MIN_PY_VERSION:
sys.stderr.write('Error, Onionr requires Python 3.%s+' % (MIN_PY_VERSION,))
@ -40,10 +44,6 @@ try:
except ImportError:
raise Exception("You need the PySocks module (for use with socks5 proxy to use Tor)")
ONIONR_TAGLINE = 'Private P2P Communication - GPLv3 - https://Onionr.net'
ONIONR_VERSION = '0.0.0' # for debugging and stuff
ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION)
API_VERSION = '0' # increments of 1; only change when something fundamental about how the API works changes. This way other nodes know how to communicate without learning too much information about you.
class Onionr:
def __init__(self):
@ -72,7 +72,7 @@ class Onionr:
data_exists = Onionr.setupConfig(self.dataDir, self)
if netcontroller.torBinary() is None:
logger.error('Tor is not installed')
logger.error('Tor is not installed', terminal=True)
sys.exit(1)
# If block data folder does not exist
@ -101,7 +101,6 @@ class Onionr:
self.onionrCore = core.Core()
self.onionrCore.onionrInst = self
#self.deleteRunFiles()
self.onionrUtils = onionrutils.OnionrUtils(self.onionrCore)
self.clientAPIInst = '' # Client http api instance
self.publicAPIInst = '' # Public http api instance

3
onionr/onionrblockapi.py
View File

@ -21,6 +21,7 @@
import core as onionrcore, logger, config, onionrexceptions, nacl.exceptions
import json, os, sys, datetime, base64, onionrstorage
from onionrusers import onionrusers
from onionrutils import stringvalidators
class Block:
blockCacheOrder = list() # NEVER write your own code that writes to this!
@ -441,7 +442,7 @@ class Block:
'''
try:
if (not self.isSigned()) or (not self.getCore()._utils.validatePubKey(signer)):
if (not self.isSigned()) or (not stringvalidators.validate_pub_key(signer)):
return False
return bool(self.getCore()._crypto.edVerify(self.getSignedData(), signer, self.getSignature(), encodedData = encodedData))

2
onionr/onionrcommands/keyadders.py
View File

@ -25,7 +25,7 @@ def add_peer(o_inst):
except IndexError:
pass
else:
if o_inst.onionrUtils.hasKey(newPeer):
if newPeer in o_inst.onionrCore.listPeers():
logger.info('We already have that key', terminal=True)
return
logger.info("Adding peer: " + logger.colors.underline + newPeer, terminal=True)

4
onionr/onionrcommands/onionrstatistics.py
View File

@ -21,7 +21,7 @@ import os, uuid, time
import logger, onionrutils
from onionrblockapi import Block
import onionr
from onionrutils import checkcommunicator
from onionrutils import checkcommunicator, mnemonickeys
def show_stats(o_inst):
try:
@ -87,7 +87,7 @@ def show_details(o_inst):
'Node Address' : o_inst.get_hostname(),
'Web Password' : o_inst.getWebPassword(),
'Public Key' : o_inst.onionrCore._crypto.pubKey,
'Human-readable Public Key' : o_inst.onionrCore._utils.getHumanReadableID()
'Human-readable Public Key' : mnemonickeys.get_human_readable_ID(o_inst.onionrCore)
}
for detail in details:

5
onionr/onionrcommands/pubkeymanager.py
View File

@ -20,6 +20,7 @@
import sys, getpass
import logger, onionrexceptions
from onionrutils import stringvalidators
from onionrusers import onionrusers, contactmanager
import unpaddedbase32
def add_ID(o_inst):
@ -55,7 +56,7 @@ def change_ID(o_inst):
except IndexError:
logger.warn('Specify pubkey to use', terminal=True)
else:
if o_inst.onionrUtils.validatePubKey(key):
if stringvalidators.validate_pub_key(key):
if key in o_inst.onionrCore._crypto.keyManager.getPubkeyList():
o_inst.onionrCore.config.set('general.public_key', key)
o_inst.onionrCore.config.save()
@ -82,7 +83,7 @@ def friend_command(o_inst):
elif action in ('add', 'remove'):
try:
friend = sys.argv[3]
if not o_inst.onionrUtils.validatePubKey(friend):
if not stringvalidators.validate_pub_key(friend):
raise onionrexceptions.InvalidPubkey('Public key is invalid')
if friend not in o_inst.onionrCore.listPeers():
raise onionrexceptions.KeyNotKnown

21
onionr/onionrcrypto.py
View File

@ -21,7 +21,8 @@ import os, binascii, base64, hashlib, time, sys, hmac, secrets
import nacl.signing, nacl.encoding, nacl.public, nacl.hash, nacl.pwhash, nacl.utils, nacl.secret
import unpaddedbase32
import logger, onionrproofs
import onionrexceptions, keymanager, core
from onionrutils import stringvalidators
import onionrexceptions, keymanager, core, onionrutils
import config
config.reload()
@ -38,8 +39,8 @@ class OnionrCrypto:
# Load our own pub/priv Ed25519 keys, gen & save them if they don't exist
if os.path.exists(self._keyFile):
if len(config.get('general.public_key', '')) > 0:
self.pubKey = config.get('general.public_key')
if len(self._core.config.get('general.public_key', '')) > 0:
self.pubKey = self._core.config.get('general.public_key')
else:
self.pubKey = self.keyManager.getPubkeyList()[0]
self.privKey = self.keyManager.getPrivkey(self.pubKey)
@ -94,10 +95,10 @@ class OnionrCrypto:
def pubKeyEncrypt(self, data, pubkey, encodedData=False):
'''Encrypt to a public key (Curve25519, taken from base32 Ed25519 pubkey)'''
pubkey = unpaddedbase32.repad(self._core._utils.strToBytes(pubkey))
pubkey = unpaddedbase32.repad(onionrutils.str_to_bytes(pubkey))
retVal = ''
box = None
data = self._core._utils.strToBytes(data)
data = onionrutils.str_to_bytes(data)
pubkey = nacl.signing.VerifyKey(pubkey, encoder=nacl.encoding.Base32Encoder()).to_curve25519_public_key()
@ -122,7 +123,7 @@ class OnionrCrypto:
privkey = self.privKey
ownKey = nacl.signing.SigningKey(seed=privkey, encoder=nacl.encoding.Base32Encoder()).to_curve25519_private_key()
if self._core._utils.validatePubKey(privkey):
if stringvalidators.validate_pub_key(privkey):
privkey = nacl.signing.SigningKey(seed=privkey, encoder=nacl.encoding.Base32Encoder()).to_curve25519_private_key()
anonBox = nacl.public.SealedBox(privkey)
else:
@ -181,7 +182,7 @@ class OnionrCrypto:
def generateDeterministic(self, passphrase, bypassCheck=False):
'''Generate a Ed25519 public key pair from a password'''
passStrength = self.deterministicRequirement
passphrase = self._core._utils.strToBytes(passphrase) # Convert to bytes if not already
passphrase = onionrutils.str_to_bytes(passphrase) # Convert to bytes if not already
# Validate passphrase length
if not bypassCheck:
if len(passphrase) < passStrength:
@ -201,7 +202,7 @@ class OnionrCrypto:
if pubkey == '':
pubkey = self.pubKey
prev = ''
pubkey = self._core._utils.strToBytes(pubkey)
pubkey = onionrutils.str_to_bytes(pubkey)
for i in range(self.HASH_ID_ROUNDS):
try:
prev = prev.encode()
@ -250,8 +251,8 @@ class OnionrCrypto:
difficulty = onionrproofs.getDifficultyForNewBlock(blockContent, ourBlock=False, coreInst=self._core)
if difficulty < int(config.get('general.minimum_block_pow')):
difficulty = int(config.get('general.minimum_block_pow'))
if difficulty < int(self._core.config.get('general.minimum_block_pow')):
difficulty = int(self._core.config.get('general.minimum_block_pow'))
mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode()
puzzle = mainHash[:difficulty]

7
onionr/onionrproofs.py
View File

@ -18,7 +18,8 @@
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import multiprocessing, nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, sys, json
import core, onionrutils, config, logger, onionrblockapi
import core, config, logger, onionrblockapi
from onionrutils import bytesconverter
config.reload()
@ -31,8 +32,6 @@ def getDifficultyModifier(coreOrUtilsInst=None):
retData = 0
if isinstance(classInst, core.Core):
useFunc = classInst._utils.storageCounter.getPercent
elif isinstance(classInst, onionrutils.OnionrUtils):
useFunc = classInst.storageCounter.getPercent
else:
useFunc = core.Core()._utils.storageCounter.getPercent
@ -56,7 +55,7 @@ def getDifficultyForNewBlock(data, ourBlock=True, coreInst=None):
if isinstance(data, onionrblockapi.Block):
dataSize = len(data.getRaw().encode('utf-8'))
else:
dataSize = len(onionrutils.OnionrUtils.strToBytes(data))
dataSize = len(bytesconverter.str_to_bytes(data))
if ourBlock:
minDifficulty = config.get('general.minimum_send_pow', 4)

2
onionr/onionrservices/bootstrapservice.py
View File

@ -33,7 +33,7 @@ def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=300):
if core_inst is None:
core_inst = core.Core()
if not core_inst._utils.validatePubKey(peer):
if not stringvalidators.validate_pub_key(peer):
raise ValueError('Peer must be valid base32 ed25519 public key')
bootstrap_port = getOpenPort()

3
onionr/onionrservices/connectionserver.py
View File

@ -24,6 +24,7 @@ import core, logger, httpapi
import onionrexceptions
from netcontroller import getOpenPort
import api
from onionrutils import stringvalidators
from . import httpheaders
class ConnectionServer:
@ -33,7 +34,7 @@ class ConnectionServer:
else:
self.core_inst = core_inst
if not core_inst._utils.validatePubKey(peer):
if not stringvalidators.validate_pub_key(peer):
raise ValueError('Peer must be valid base32 ed25519 public key')
socks = core_inst.config.get('tor.socksport') # Load config for Tor socks port for proxy

3
onionr/onionrstorage/__init__.py
View File

@ -18,6 +18,7 @@
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import core, sys, sqlite3, os, dbcreator, onionrexceptions
from onionrutils import bytesconverter
DB_ENTRY_SIZE_LIMIT = 10000 # Will be a config option
@ -82,7 +83,7 @@ def getData(coreInst, bHash):
assert isinstance(coreInst, core.Core)
assert coreInst._utils.validateHash(bHash)
bHash = coreInst._utils.bytesToStr(bHash)
bHash = bytesconverter.bytes_to_str(bHash)
# First check DB for data entry by hash
# if no entry, check disk

3
onionr/onionrusers/contactmanager.py
View File

@ -20,10 +20,11 @@
import os, json, onionrexceptions
import unpaddedbase32
from onionrusers import onionrusers
from onionrutils import bytesconverter
class ContactManager(onionrusers.OnionrUser):
def __init__(self, coreInst, publicKey, saveUser=False, recordExpireSeconds=5):
publicKey = unpaddedbase32.repad(coreInst._utils.strToBytes(publicKey)).decode()
publicKey = unpaddedbase32.repad(bytesconverter.str_to_bytes(publicKey)).decode()
super(ContactManager, self).__init__(coreInst, publicKey, saveUser=saveUser)
self.dataDir = coreInst.dataDir + '/contacts/'
self.dataFile = '%s/contacts/%s.json' % (coreInst.dataDir, publicKey)

12
onionr/onionrusers/onionrusers.py
View File

@ -17,7 +17,9 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import onionrblockapi, logger, onionrexceptions, json, sqlite3, time
import logger, onionrexceptions, json, sqlite3, time
from onionrutils import stringvalidators, bytesconverter
import unpaddedbase32
import nacl.exceptions
@ -56,7 +58,7 @@ class OnionrUser:
Takes an instance of onionr core, a base32 encoded ed25519 public key, and a bool saveUser
saveUser determines if we should add a user to our peer database or not.
'''
publicKey = unpaddedbase32.repad(coreInst._utils.strToBytes(publicKey)).decode()
publicKey = unpaddedbase32.repad(bytesconverter.str_to_bytes(publicKey)).decode()
self.trust = 0
self._core = coreInst
@ -103,7 +105,7 @@ class OnionrUser:
deleteExpiredKeys(self._core)
retData = ''
forwardKey = self._getLatestForwardKey()
if self._core._utils.validatePubKey(forwardKey[0]):
if stringvalidators.validate_pub_key(forwardKey[0]):
retData = self._core._crypto.pubKeyEncrypt(data, forwardKey[0], encodedData=True)
else:
raise onionrexceptions.InvalidPubkey("No valid forward secrecy key available for this user")
@ -190,8 +192,8 @@ class OnionrUser:
return list(keyList)
def addForwardKey(self, newKey, expire=DEFAULT_KEY_EXPIRE):
newKey = self._core._utils.bytesToStr(unpaddedbase32.repad(self._core._utils.strToBytes(newKey)))
if not self._core._utils.validatePubKey(newKey):
newKey = self._core._utils.bytesToStr(unpaddedbase32.repad(bytesconverter.str_to_bytes(newKey)))
if not stringvalidators.validate_pub_key(newKey):
# Do not add if something went wrong with the key
raise onionrexceptions.InvalidPubkey(newKey)

106
onionr/onionrutils/__init__.py Executable file → Normal file
View File

@ -22,13 +22,11 @@ import sys, os, sqlite3, binascii, time, base64, json, glob, shutil, math, re, u
import requests
import nacl.signing, nacl.encoding
import unpaddedbase32
from onionrblockapi import Block
import onionrexceptions, config, logger
import onionrevents
import storagecounter
from etc import pgpwords, onionrvalues
from onionrusers import onionrusers
from . import localcommand, blockmetadata, validatemetadata, basicrequests
from . import localcommand, blockmetadata, basicrequests, validatemetadata
from . import stringvalidators
config.reload()
@ -45,39 +43,6 @@ class OnionrUtils:
self.storageCounter = storagecounter.StorageCounter(self._core) # used to keep track of how much data onionr is using on disk
return
def getRoundedEpoch(self, roundS=60):
'''
Returns the epoch, rounded down to given seconds (Default 60)
'''
epoch = self.getEpoch()
return epoch - (epoch % roundS)
def getHumanReadableID(self, pub=''):
'''gets a human readable ID from a public key'''
if pub == '':
pub = self._core._crypto.pubKey
pub = base64.b16encode(base64.b32decode(pub)).decode()
return ' '.join(pgpwords.wordify(pub))
def convertHumanReadableID(self, pub):
'''Convert a human readable pubkey id to base32'''
pub = pub.lower()
return self.bytesToStr(base64.b32encode(binascii.unhexlify(pgpwords.hexify(pub.strip()))))
def getBlockMetadataFromData(self, blockData):
'''
accepts block contents as string, returns a tuple of
metadata, meta (meta being internal metadata, which will be
returned as an encrypted base64 string if it is encrypted, dict if not).
'''
return blockmetadata.get_block_metadata_from_data(self, blockData)
def processBlockMetadata(self, blockHash):
'''
Read metadata from a block and cache it to the block database
'''
return blockmetadata.process_block_metadata(self, blockHash)
def escapeAnsi(self, line):
'''
Remove ANSI escape codes from a string with regex
@ -88,46 +53,12 @@ class OnionrUtils:
ansi_escape = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -/]*[@-~]')
return ansi_escape.sub('', line)
def hasBlock(self, hash):
'''
Check for new block in the list
'''
conn = sqlite3.connect(self._core.blockDB)
c = conn.cursor()
if not self.validateHash(hash):
raise Exception("Invalid hash")
for result in c.execute("SELECT COUNT() FROM hashes WHERE hash = ?", (hash,)):
if result[0] >= 1:
conn.commit()
conn.close()
return True
else:
conn.commit()
conn.close()
return False
def hasKey(self, key):
'''
Check for key in list of public keys
'''
return key in self._core.listPeers()
def validateHash(self, data, length=64):
'''
Validate if a string is a valid hash hex digest (does not compare, just checks length and charset)
'''
return stringvalidators.validate_hash(self, data, length)
def validateMetadata(self, metadata, blockData):
'''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string'''
return validatemetadata.validate_metadata(self, metadata, blockData)
def validatePubKey(self, key):
'''
Validate if a string is a valid base32 encoded Ed25519 key
'''
return stringvalidators.validate_pub_key(self, key)
def getEpoch(self):
'''returns epoch'''
return math.floor(time.time())
@ -144,21 +75,6 @@ class OnionrUtils:
'''
return basicrequests.do_get_request(self, url, port, proxyType, ignoreAPI, returnHeaders)
@staticmethod
def strToBytes(data):
try:
data = data.encode()
except AttributeError:
pass
return data
@staticmethod
def bytesToStr(data):
try:
data = data.decode()
except AttributeError:
pass
return data
def size(path='.'):
'''
Returns the size of a folder's contents in bytes
@ -183,4 +99,22 @@ def humanSize(num, suffix='B'):
if abs(num) < 1024.0:
return "%.1f %s%s" % (num, unit, suffix)
num /= 1024.0
return "%.1f %s%s" % (num, 'Yi', suffix)
return "%.1f %s%s" % (num, 'Yi', suffix)
def has_block(core_inst, hash):
'''
Check for new block in the list
'''
conn = sqlite3.connect(core_inst.blockDB)
c = conn.cursor()
if not stringvalidators.validate_hash(hash):
raise Exception("Invalid hash")
for result in c.execute("SELECT COUNT() FROM hashes WHERE hash = ?", (hash,)):
if result[0] >= 1:
conn.commit()
conn.close()
return True
else:
conn.commit()
conn.close()
return False

2
onionr/onionrutils/basicrequests.py
View File

@ -1,6 +1,5 @@
import requests
import logger, onionrexceptions
from onionr import API_VERSION
def do_post_request(utils_inst, url, data={}, port=0, proxyType='tor'):
'''
Do a POST request through a local tor or i2p instance
@ -29,6 +28,7 @@ def do_get_request(utils_inst, url, port=0, proxyType='tor', ignoreAPI=False, re
'''
Do a get request through a local tor or i2p instance
'''
API_VERSION = utils_inst._core.onionrInst.API_VERSION
retData = False
if proxyType == 'tor':
if port == 0:

9
onionr/onionrutils/blockmetadata.py
View File

@ -2,8 +2,9 @@ import json
import logger, onionrevents
from onionrusers import onionrusers
from etc import onionrvalues
from onionrblockapi import Block
def get_block_metadata_from_data(utils_inst, blockData):
import onionrblockapi
from . import epoch
def get_block_metadata_from_data(blockData):
'''
accepts block contents as string, returns a tuple of
metadata, meta (meta being internal metadata, which will be
@ -36,8 +37,8 @@ def process_block_metadata(utils_inst, blockHash):
'''
Read metadata from a block and cache it to the block database
'''
curTime = utils_inst.getRoundedEpoch(roundS=60)
myBlock = Block(blockHash, utils_inst._core)
curTime = epoch.get_rounded_epoch(roundS=60)
myBlock = onionrblockapi.Block(blockHash, utils_inst._core)
if myBlock.isEncrypted:
myBlock.decrypt()
if (myBlock.isEncrypted and myBlock.decrypted) or (not myBlock.isEncrypted):

13
onionr/onionrutils/bytesconverter.py Normal file
View File

@ -0,0 +1,13 @@
def str_to_bytes(data):
try:
data = data.encode()
except AttributeError:
pass
return data
def bytes_to_str(data):
try:
data = data.decode()
except AttributeError:
pass
return data

11
onionr/onionrutils/epoch.py Normal file
View File

@ -0,0 +1,11 @@
import math, time
def get_rounded_epoch(roundS=60):
'''
Returns the epoch, rounded down to given seconds (Default 60)
'''
epoch = get_epoch()
return epoch - (epoch % roundS)
def get_epoch(self):
'''returns epoch'''
return math.floor(time.time())

3
onionr/onionrutils/importnewblocks.py
View File

@ -1,5 +1,6 @@
import glob
import logger, core
from onionrutils import blockmetadata
def import_new_blocks(core_inst=None, scanDir=''):
'''
This function is intended to scan for new blocks ON THE DISK and import them
@ -21,7 +22,7 @@ def import_new_blocks(core_inst=None, scanDir=''):
if core_inst._crypto.sha3Hash(newBlock.read()) == block.replace('.dat', ''):
core_inst.addToBlockDB(block.replace('.dat', ''), dataSaved=True)
logger.info('Imported block %s.' % block)
core_inst._utils.processBlockMetadata(block)
blockmetadata.process_block_metadata(block)
else:
logger.warn('Failed to verify hash for %s' % block)
if not exist:

8
onionr/onionrutils/mnemonickeys.py Normal file
View File

@ -0,0 +1,8 @@
import base64
from etc import pgpwords
def get_human_readable_ID(core_inst, pub=''):
'''gets a human readable ID from a public key'''
if pub == '':
pub = core_inst._crypto.pubKey
pub = base64.b16encode(base64.b32decode(pub)).decode()
return ' '.join(pgpwords.wordify(pub))

6
onionr/onionrutils/stringvalidators.py
View File

@ -1,4 +1,4 @@
import base64, string
import base64, string, onionrutils
import unpaddedbase32, nacl.signing, nacl.encoding
def validate_hash(utils_inst, data, length=64):
'''
@ -18,14 +18,14 @@ def validate_hash(utils_inst, data, length=64):
return retVal
def validate_pub_key(utils_inst, key):
def validate_pub_key(key):
'''
Validate if a string is a valid base32 encoded Ed25519 key
'''
if type(key) is type(None):
return False
# Accept keys that have no = padding
key = unpaddedbase32.repad(utils_inst.strToBytes(key))
key = unpaddedbase32.repad(onionrutils.str_to_bytes(key))
retVal = False
try:

20
onionr/onionrutils/validatemetadata.py
View File

@ -2,7 +2,7 @@ import json
import logger, onionrexceptions
from etc import onionrvalues
from onionrutils import stringvalidators
def validate_metadata(utils_inst, metadata, blockData):
def validate_metadata(core_inst, metadata, blockData):
'''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string'''
# TODO, make this check sane sizes
retData = False
@ -16,11 +16,11 @@ def validate_metadata(utils_inst, metadata, blockData):
pass
# Validate metadata dict for invalid keys to sizes that are too large
maxAge = utils_inst._core.config.get("general.max_block_age", onionrvalues.OnionrValues().default_expire)
maxAge = core_inst.config.get("general.max_block_age", onionrvalues.OnionrValues().default_expire)
if type(metadata) is dict:
for i in metadata:
try:
utils_inst._core.requirements.blockMetadataLengths[i]
core_inst.requirements.blockMetadataLengths[i]
except KeyError:
logger.warn('Block has invalid metadata key ' + i)
break
@ -30,25 +30,25 @@ def validate_metadata(utils_inst, metadata, blockData):
testData = len(testData)
except (TypeError, AttributeError) as e:
testData = len(str(testData))
if utils_inst._core.requirements.blockMetadataLengths[i] < testData:
if core_inst.requirements.blockMetadataLengths[i] < testData:
logger.warn('Block metadata key ' + i + ' exceeded maximum size')
break
if i == 'time':
if not stringvalidators.is_integer_string(metadata[i]):
logger.warn('Block metadata time stamp is not integer string or int')
break
isFuture = (metadata[i] - utils_inst.getEpoch())
isFuture = (metadata[i] - core_inst.getEpoch())
if isFuture > maxClockDifference:
logger.warn('Block timestamp is skewed to the future over the max %s: %s' (maxClockDifference, isFuture))
break
if (utils_inst.getEpoch() - metadata[i]) > maxAge:
if (core_inst.getEpoch() - metadata[i]) > maxAge:
logger.warn('Block is outdated: %s' % (metadata[i],))
break
elif i == 'expire':
try:
assert int(metadata[i]) > utils_inst.getEpoch()
assert int(metadata[i]) > core_inst.getEpoch()
except AssertionError:
logger.warn('Block is expired: %s less than %s' % (metadata[i], utils_inst.getEpoch()))
logger.warn('Block is expired: %s less than %s' % (metadata[i], core_inst.getEpoch()))
break
elif i == 'encryptType':
try:
@ -59,9 +59,9 @@ def validate_metadata(utils_inst, metadata, blockData):
else:
# if metadata loop gets no errors, it does not break, therefore metadata is valid
# make sure we do not have another block with the same data content (prevent data duplication and replay attacks)
nonce = utils_inst._core._utils.bytesToStr(utils_inst._core._crypto.sha3Hash(blockData))
nonce = core_inst._utils.bytesToStr(core_inst._crypto.sha3Hash(blockData))
try:
with open(utils_inst._core.dataNonceFile, 'r') as nonceFile:
with open(core_inst.dataNonceFile, 'r') as nonceFile:
if nonce in nonceFile.read():
retData = False # we've seen that nonce before, so we can't pass metadata
raise onionrexceptions.DataExists

3
onionr/static-data/default-plugins/encrypt/main.py
View File

@ -21,6 +21,7 @@
# Imports some useful libraries
import logger, config, threading, time, datetime, sys, json
from onionrblockapi import Block
from onionrutils import stringvalidators
import onionrexceptions, onionrusers
import locale
locale.setlocale(locale.LC_ALL, '')
@ -43,7 +44,7 @@ class PlainEncryption:
pass
try:
if not self.api.get_core()._utils.validatePubKey(sys.argv[2]):
if not stringvalidators.validate_pub_key(sys.argv[2]):
raise onionrexceptions.InvalidPubkey
except (ValueError, IndexError) as e:
logger.error("Peer public key not specified", terminal=True)

3
onionr/static-data/default-plugins/esoteric/main.py
View File

@ -23,6 +23,7 @@ import locale, sys, os, threading, json
locale.setlocale(locale.LC_ALL, '')
import onionrservices, logger
from onionrservices import bootstrapservice
from onionrutils import stringvalidators
plugin_name = 'esoteric'
PLUGIN_VERSION = '0.0.0'
@ -66,7 +67,7 @@ class Esoteric:
def create(self):
try:
peer = sys.argv[2]
if not self.myCore._utils.validatePubKey(peer):
if not stringvalidators.validate_pub_key(peer):
exit_with_error('Invalid public key specified')
except IndexError:
exit_with_error('You must specify a peer public key')

3
onionr/static-data/default-plugins/metadataprocessor/main.py
View File

@ -23,6 +23,7 @@ import logger, config
import os, sys, json, time, random, shutil, base64, getpass, datetime, re
from onionrblockapi import Block
import onionrusers, onionrexceptions
from onionrutils import stringvalidators
plugin_name = 'metadataprocessor'
@ -36,7 +37,7 @@ def _processForwardKey(api, myBlock):
key = myBlock.getMetadata('newFSKey')
# We don't need to validate here probably, but it helps
if api.get_utils().validatePubKey(key):
if stringvalidators.validate_pub_key(key):
peer.addForwardKey(key)
else:
raise onionrexceptions.InvalidPubkey("%s is not a valid pubkey key" % (key,))

8
onionr/static-data/default-plugins/pluginmanager/main.py
View File

@ -22,7 +22,7 @@
import logger, config
import os, sys, json, time, random, shutil, base64, getpass, datetime, re
from onionrblockapi import Block
from onionrutils import importnewblocks
from onionrutils import importnewblocks, stringvalidators,
plugin_name = 'pluginmanager'
@ -399,13 +399,13 @@ def commandInstallPlugin():
valid_hash = pluginapi.get_utils().validateHash(pkobh)
real_block = False
valid_key = pluginapi.get_utils().validatePubKey(pkobh)
valid_key = stringvalidators.validate_pub_key(pkobh)
real_key = False
if valid_hash:
real_block = Block.exists(pkobh)
elif valid_key:
real_key = pluginapi.get_utils().hasKey(pkobh)
real_key = pkobh in pluginapi.get_core().listPeers()
blockhash = None
@ -493,7 +493,7 @@ def commandAddRepository():
pluginslist = dict()
for pluginname, distributor in blockContent['plugins']:
if pluginapi.get_utils().validatePubKey(distributor):
if stringvalidators.validate_pub_key(distributor):
pluginslist[pluginname] = distributor
logger.debug('Found %s records in repository.' % len(pluginslist), terminal=True)

3
onionr/static-data/default-plugins/pms/main.py
View File

@ -23,6 +23,7 @@ import logger, config, threading, time, datetime
from onionrblockapi import Block
import onionrexceptions
from onionrusers import onionrusers
from onionrutils import stringvalidators
import locale, sys, os, json
locale.setlocale(locale.LC_ALL, '')
@ -217,7 +218,7 @@ class OnionrMail:
recip = logger.readline('Enter peer address, or -q to stop:').strip()
if recip in ('-q', 'q'):
raise EOFError
if not self.myCore._utils.validatePubKey(recip):
if not stringvalidators.validate_pub_key(recip):
raise onionrexceptions.InvalidPubkey('Must be a valid ed25519 base32 encoded public key')
except onionrexceptions.InvalidPubkey:
logger.warn('Invalid public key', terminal=True)

5
onionr/subprocesspow.py
View File

@ -23,6 +23,7 @@ import subprocess, os
import multiprocessing, threading, time, json
from multiprocessing import Pipe, Process
import core, onionrblockapi, config, onionrutils, logger, onionrproofs
from onionrutils import bytesconverter
class SubprocessPOW:
def __init__(self, data, metadata, core_inst=None, subproc_count=None):
@ -51,7 +52,7 @@ class SubprocessPOW:
# dump dict to measure bytes of json metadata. Cannot reuse later because the pow token must be added
json_metadata = json.dumps(metadata).encode()
self.data = onionrutils.OnionrUtils.strToBytes(data)
self.data = bytesconverter.str_to_bytes(data)
# Calculate difficulty. Dumb for now, may use good algorithm in the future.
self.difficulty = onionrproofs.getDifficultyForNewBlock(bytes(json_metadata + b'\n' + self.data), coreInst=self.core_inst)
@ -111,7 +112,7 @@ class SubprocessPOW:
payload = json.dumps(metadata).encode() + b'\n' + data
# Check sha3_256 hash of block, compare to puzzle. Send payload if puzzle finished
token = mcore._crypto.sha3Hash(payload)
token = onionrutils.OnionrUtils.bytesToStr(token) # ensure token is string
token = bytesconverter.bytes_to_str(token) # ensure token is string
if puzzle == token[0:difficulty]:
pipe.send(payload)
break

16
onionr/tests/test_highlevelcrypto.py
View File

@ -4,6 +4,7 @@ sys.path.append(".")
import unittest, uuid, hashlib, base64
import nacl.exceptions
import nacl.signing, nacl.hash, nacl.encoding
from onionrutils import stringvalidators, mnemonickeys
TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
print("Test directory:", TEST_DIR)
os.environ["ONIONR_HOME"] = TEST_DIR
@ -45,19 +46,12 @@ class OnionrCryptoTests(unittest.TestCase):
self.assertEqual(crypto.sha3Hash(b'test'), normal)
def valid_default_id(self):
self.assertTrue(c._utils.validatePubKey(crypto.pubKey))
self.assertTrue(stringvalidators.validate_pub_key(crypto.pubKey))
def test_human_readable_length(self):
human = c._utils.getHumanReadableID()
human = mnemonickeys.get_human_readable_ID(c)
self.assertTrue(len(human.split(' ')) == 32)
def test_human_readable_rebuild(self):
return # Broken right now
# Test if we can get the human readable id, and convert it back to valid base32 key
human = c._utils.getHumanReadableID()
unHuman = c._utils.convertHumanReadableID(human)
nacl.signing.VerifyKey(c._utils.convertHumanReadableID(human), encoder=nacl.encoding.Base32Encoder)
def test_safe_compare(self):
self.assertTrue(crypto.safeCompare('test', 'test'))
self.assertTrue(crypto.safeCompare('test', b'test'))
@ -130,7 +124,7 @@ class OnionrCryptoTests(unittest.TestCase):
def test_deterministic(self):
password = os.urandom(32)
gen = crypto.generateDeterministic(password)
self.assertTrue(c._utils.validatePubKey(gen[0]))
self.assertTrue(stringvalidators.validate_pub_key(gen[0]))
try:
crypto.generateDeterministic('weakpassword')
except onionrexceptions.PasswordStrengthError:
@ -151,6 +145,6 @@ class OnionrCryptoTests(unittest.TestCase):
gen2 = crypto.generateDeterministic(password)
self.assertFalse(gen == gen1)
self.assertTrue(gen1 == gen2)
self.assertTrue(c._utils.validatePubKey(gen1[0]))
self.assertTrue(stringvalidators.validate_pub_key(gen1[0]))
unittest.main()