kev
/
Onionr
1
0
Fork 0
Code Issues 24 Pull Requests Projects Releases 2 Wiki Activity

progress in removing core

This commit is contained in:
Kevin Froman 2019-07-22 00:24:42 -05:00
parent 26c3b519c7
commit a74f2c5051
18 changed files with 83 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
onionr/communicator/__init__.py
View File

@ -214,7 +214,7 @@ class OnionrCommunicatorDaemon:
def peerCleanup(self):
'''This just calls onionrpeers.cleanupPeers, which removes dead or bad peers (offline too long, too slow)'''
onionrpeers.peer_cleanup()
onionrpeers.peer_cleanup(self.onionrInst)
self.decrementThreadCount('peerCleanup')
def getPeerProfileInstance(self, peer):

6
onionr/communicatorutils/announcenode.py
View File

@ -50,7 +50,7 @@ def announce_node(daemon):
combinedNodes = ourID + peer
if ourID != 1:
existingRand = bytesconverter.bytes_to_str(keydb.addressinfo.get_address_info(peer, 'powValue'))
existingRand = bytesconverter.bytes_to_str(keydb.transportinfo.get_address_info(peer, 'powValue'))
# Reset existingRand if it no longer meets the minimum POW
if type(existingRand) is type(None) or not existingRand.endswith('0' * ov.announce_pow):
existingRand = ''
@ -76,7 +76,7 @@ def announce_node(daemon):
if basicrequests.do_post_request(url, data) == 'Success':
logger.info('Successfully introduced node to ' + peer, terminal=True)
retData = True
keydb.addressinfo.set_address_info(peer, 'introduced', 1)
keydb.addressinfo.set_address_info(peer, 'powValue', data['random'])
keydb.transportinfo.set_address_info(peer, 'introduced', 1)
keydb.transportinfo.set_address_info(peer, 'powValue', data['random'])
daemon.decrementThreadCount('announce_node')
return retData

5
onionr/communicatorutils/connectnewpeers.py
View File

@ -64,7 +64,8 @@ def connect_new_peer_to_communicator(comm_inst, peer='', useBootstrap=False):
if comm_inst.shutdown:
return
# Ping a peer,
if peeraction.peer_action(comm_inst, address, 'ping') == 'pong!':
ret = peeraction.peer_action(comm_inst, address, 'ping')
if ret == 'pong!':
time.sleep(0.1)
if address not in mainPeerList:
# Add a peer to our list if it isn't already since it successfully connected
@ -85,5 +86,5 @@ def connect_new_peer_to_communicator(comm_inst, peer='', useBootstrap=False):
else:
# Mark a peer as tried if they failed to respond to ping
tried.append(address)
logger.debug('Failed to connect to ' + address)
logger.debug('Failed to connect to %s: %s ' % (address, ret))
return retData

9
onionr/communicatorutils/downloadblocks/__init__.py
View File

@ -20,13 +20,12 @@
import communicator, onionrexceptions
import logger, onionrpeers
from onionrutils import blockmetadata, stringvalidators, validatemetadata
from coredb import blockmetadb
from . import shoulddownload
from communicator import peeraction, onlinepeers
import onionrcrypto, onionrstorage, onionrblacklist, storagecounter
def download_blocks_from_communicator(comm_inst):
assert isinstance(comm_inst, communicator.OnionrCommunicatorDaemon)
crypto = onionrcrypto.OnionrCrypto()
blacklist = onionrblacklist.OnionrBlackList()
storage_counter = storagecounter.StorageCounter()
for blockHash in list(comm_inst.blockQueue):
@ -54,7 +53,7 @@ def download_blocks_from_communicator(comm_inst):
if len(blockPeers) == 0:
peerUsed = onlinepeers.pick_online_peer(comm_inst)
else:
blockPeers = crypto.randomShuffle(blockPeers)
blockPeers = onionrcrypto.cryptoutils.random_shuffle(blockPeers)
peerUsed = blockPeers.pop(0)
if not comm_inst.shutdown and peerUsed.strip() != '':
@ -66,7 +65,7 @@ def download_blocks_from_communicator(comm_inst):
except AttributeError:
pass
realHash = ccrypto.sha3Hash(content)
realHash = onionrcrypto.hashers.sha3_hash(content)
try:
realHash = realHash.decode() # bytes on some versions for some reason
except AttributeError:
@ -76,7 +75,7 @@ def download_blocks_from_communicator(comm_inst):
metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
metadata = metas[0]
if validatemetadata.validate_metadata(metadata, metas[2]): # check if metadata is valid, and verify nonce
if crypto.verifyPow(content): # check if POW is enough/correct
if onionrcrypto.cryptoutils.verify_POW(content): # check if POW is enough/correct
logger.info('Attempting to save block %s...' % blockHash[:12])
try:
onionrstorage.setdata.set_data(content)

4
onionr/communicatorutils/downloadblocks/shoulddownload.py
View File

@ -18,12 +18,14 @@
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
from coredb import blockmetadb
import onionrblacklist
def should_download(comm_inst, block_hash):
blacklist = onionrblacklist.OnionrBlackList()
ret_data = True
if block_hash in blockmetadb.get_block_list(): # Dont download block we have
ret_data = False
else:
if comm_inst.blacklist.inBlacklist(block_hash): # Dont download blacklisted block
if blacklist.inBlacklist(block_hash): # Dont download blacklisted block
ret_data = False
if ret_data is False:
# Remove block from communicator queue if it shouldnt be downloaded

3
onionr/communicatorutils/lookupadders.py
View File

@ -21,6 +21,7 @@ import logger
from onionrutils import stringvalidators
from communicator import peeraction, onlinepeers
from utils import gettransports
transports = gettransports.get()
def lookup_new_peer_transports_with_communicator(comm_inst):
logger.info('Looking up new addresses...')
tryAmount = 1
@ -41,7 +42,7 @@ def lookup_new_peer_transports_with_communicator(comm_inst):
invalid = []
for x in newPeers:
x = x.strip()
if not stringvalidators.validate_transport(x) or x in comm_inst.newPeers or x == gettransports.transports[0]:
if not stringvalidators.validate_transport(x) or x in comm_inst.newPeers or x in transports:
# avoid adding if its our address
invalid.append(x)
for x in invalid:

4
onionr/communicatorutils/lookupblocks.py
View File

@ -21,6 +21,8 @@ import logger, onionrproofs
from onionrutils import stringvalidators, epoch
from communicator import peeraction, onlinepeers
from coredb import blockmetadb
import onionrblacklist
blacklist = onionrblacklist.OnionrBlackList()
def lookup_blocks_from_communicator(comm_inst):
logger.info('Looking up new blocks...')
tryAmount = 2
@ -72,7 +74,7 @@ def lookup_blocks_from_communicator(comm_inst):
if not i in existingBlocks:
# if block does not exist on disk and is not already in block queue
if i not in comm_inst.blockQueue:
if onionrproofs.hashMeetsDifficulty(i) and not comm_inst.blacklist.inBlacklist(i):
if onionrproofs.hashMeetsDifficulty(i) and not blacklist.inBlacklist(i):
if len(comm_inst.blockQueue) <= 1000000:
comm_inst.blockQueue[i] = [peer] # add blocks to download queue
new_block_count += 1

4
onionr/communicatorutils/uploadblocks.py
View File

@ -22,14 +22,14 @@ from communicatorutils import proxypicker
import onionrblockapi as block
from onionrutils import localcommand, stringvalidators, basicrequests
from communicator import onlinepeers
import onionrcrypto
def upload_blocks_from_communicator(comm_inst):
# when inserting a block, we try to upload it to a few peers to add some deniability
TIMER_NAME = "upload_blocks_from_communicator"
triedPeers = []
finishedUploads = []
comm_inst.blocksToUpload = comm_inst.crypto.randomShuffle(comm_inst.blocksToUpload)
comm_inst.blocksToUpload = onionrcrypto.cryptoutils.random_shuffle(comm_inst.blocksToUpload)
if len(comm_inst.blocksToUpload) != 0:
for bl in comm_inst.blocksToUpload:
if not stringvalidators.validate_hash(bl):

2
onionr/coredb/daemonqueue/__init__.py
View File

@ -75,7 +75,7 @@ def daemon_queue_get_response(responseID=''):
'''
Get a response sent by communicator to the API, by requesting to the API
'''
if len(responseID) > 0: raise ValueError('ResponseID should not be empty')
if len(responseID) == 0: raise ValueError('ResponseID should not be empty')
resp = localcommand.local_command(dbfiles.daemon_queue_db, 'queueResponse/' + responseID)
return resp

4
onionr/coredb/keydb/listkeys.py
View File

@ -21,7 +21,7 @@ import sqlite3
import logger
from onionrutils import epoch
from .. import dbfiles
from . import userinfo
from . import userinfo, transportinfo
def list_peers(randomOrder=True, getPow=False, trust=0):
'''
Return a list of public keys (misleading function name)
@ -78,7 +78,7 @@ def list_adders(randomOrder=True, i2p=True, recent=0):
testList = list(addressList) # create new list to iterate
for address in testList:
try:
if recent > 0 and (epoch.get_epoch() - userinfo.get_user_info(address, 'lastConnect')) > recent:
if recent > 0 and (epoch.get_epoch() - transportinfo.get_address_info(address, 'lastConnect')) > recent:
raise TypeError # If there is no last-connected date or it was too long ago, don't add peer to list if recent is not 0
except TypeError:
addressList.remove(address)

8
onionr/httpapi/security/public.py
View File

@ -20,11 +20,12 @@
from flask import Blueprint, request, abort
from onionrservices import httpheaders
from onionrutils import epoch
from utils import gettransports
class PublicAPISecurity:
def __init__(self, public_api):
public_api_security_bp = Blueprint('publicapisecurity', __name__)
self.public_api_security_bp = public_api_security_bp
transports = gettransports.get()
@public_api_security_bp.before_app_request
def validate_request():
@ -32,10 +33,7 @@ class PublicAPISecurity:
# If high security level, deny requests to public (HS should be disabled anyway for Tor, but might not be for I2P)
if public_api.config.get('general.security_level', default=1) > 0:
abort(403)
if type(public_api.torAdder) is None and type(public_api.i2pAdder) is None:
# abort if our hs addresses are not known
abort(403)
if request.host not in (public_api.i2pAdder, public_api.torAdder):
if request.host not in transports:
# Disallow connection if wrong HTTP hostname, in order to prevent DNS rebinding attacks
abort(403)
public_api.hitCount += 1 # raise hit count for valid requests

5
onionr/onionrcrypto/cryptoutils/__init__.py
View File

@ -1,5 +1,6 @@
from . import safecompare, replayvalidation, randomshuffle
from . import safecompare, replayvalidation, randomshuffle, verifypow
replay_validator = replayvalidation.replay_timestamp_validation
random_shuffle = randomshuffle.random_shuffle
safe_compare = safecompare.safe_compare
safe_compare = safecompare.safe_compare
verify_POW = verifypow.verify_POW

36
onionr/onionrcrypto/cryptoutils/verifypow.py Normal file
View File

@ -0,0 +1,36 @@
from .. import hashers
import config, onionrproofs, logger
def verify_POW(blockContent):
'''
Verifies the proof of work associated with a block
'''
retData = False
dataLen = len(blockContent)
try:
blockContent = blockContent.encode()
except AttributeError:
pass
blockHash = hashers.sha3_hash(blockContent)
try:
blockHash = blockHash.decode() # bytes on some versions for some reason
except AttributeError:
pass
difficulty = onionrproofs.getDifficultyForNewBlock(blockContent, ourBlock=False)
if difficulty < int(config.get('general.minimum_block_pow')):
difficulty = int(config.get('general.minimum_block_pow'))
mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode()
puzzle = mainHash[:difficulty]
if blockHash[:difficulty] == puzzle:
# logger.debug('Validated block pow')
retData = True
else:
logger.debug("Invalid token, bad proof")
return retData

4
onionr/onionrpeers/peercleanup.py
View File

@ -21,7 +21,7 @@ import sqlite3
import logger
from onionrutils import epoch
from . import scoresortedpeerlist, peerprofiles
import onionrblacklist
import onionrblacklist, config
from coredb import keydb
def peer_cleanup(onionr_inst):
'''Removes peers who have been offline too long or score too low'''
@ -40,7 +40,7 @@ def peer_cleanup(onionr_inst):
if peerprofiles.PeerProfiles(address).score < min_score:
keydb.removekeys.remove_address(address)
try:
if (int(epoch.get_epoch()) - int(keydb.transportinfo.get_address_info(address, 'dateSeen'))) >= 600:
if (int(epoch.get_epoch()) - int(keydb.transportinfo.get_address_info(address, 'lastConnect'))) >= 600:
expireTime = 600
else:
expireTime = 86400

2
onionr/onionrproofs.py
View File

@ -94,6 +94,7 @@ class DataPOW:
self.data = data
self.threadCount = threadCount
self.rounds = 0
self.hashing = False
if forceDifficulty == 0:
dataLen = sys.getsizeof(data)
@ -189,6 +190,7 @@ class POW:
self.data = data
self.metadata = metadata
self.threadCount = threadCount
self.hashing = False
json_metadata = json.dumps(metadata).encode()

3
onionr/onionrutils/validatemetadata.py
View File

@ -25,7 +25,6 @@ import config, filepaths, onionrcrypto
def validate_metadata(metadata, blockData):
'''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string'''
# TODO, make this check sane sizes
crypto = onionrcrypto.OnionrCrypto()
requirements = onionrvalues.OnionrValues()
retData = False
maxClockDifference = 120
@ -81,7 +80,7 @@ def validate_metadata(metadata, blockData):
else:
# if metadata loop gets no errors, it does not break, therefore metadata is valid
# make sure we do not have another block with the same data content (prevent data duplication and replay attacks)
nonce = bytesconverter.bytes_to_str(crypto.sha3Hash(blockData))
nonce = bytesconverter.bytes_to_str(onionrcrypto.hashers.sha3_hash(blockData))
try:
with open(filepaths.data_nonce_file, 'r') as nonceFile:
if nonce in nonceFile.read():

0
onionr/utils/__init__.py Normal file
View File

20
onionr/utils/gettransports.py
View File

@ -1,14 +1,18 @@
import filepaths
import filepaths, time
files = [filepaths.tor_hs_address_file]
def get():
transports = []
for file in files:
try:
with open(file, 'r') as transport_file:
transports.append(transport_file.read())
except FileNotFoundError:
transports.append('')
pass
while len(transports) == 0:
for file in files:
try:
with open(file, 'r') as transport_file:
transports.append(transport_file.read().strip())
except FileNotFoundError:
transports.append('')
else:
break
else:
time.sleep(1)
return list(transports)