kev
/
Onionr
1
0
Fork 0
Code Issues 24 Pull Requests Projects Releases 2 Wiki Activity

minor bug fixes and linting improvements

This commit is contained in:
Kevin Froman 2019-12-23 01:51:24 -06:00
parent 9329b07e3b
commit 87ea8d137b
8 changed files with 223 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
src/communicator/bootstrappeers.py
View File

@ -1,9 +1,10 @@
'''
Onionr - Private P2P Communication
"""Onionr - Private P2P Communication.
add bootstrap peers to the communicator peer list
'''
'''
add bootstrap peers to the communicator peer list
"""
from utils import readstatic, gettransports
from coredb import keydb
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@ -16,16 +17,16 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
from utils import readstatic, gettransports
from coredb import keydb
"""
bootstrap_peers = readstatic.read_static('bootstrap-nodes.txt').split(',')
def add_bootstrap_list_to_peer_list(comm_inst, peerList, db_only=False):
'''
Add the bootstrap list to the peer list (no duplicates)
'''
"""Add the bootstrap list to the peer list (no duplicates)."""
for i in bootstrap_peers:
if i not in peerList and i not in comm_inst.offlinePeers and not i in gettransports.get() and len(str(i).strip()) > 0:
if not db_only: peerList.append(i)
if i not in peerList and i not in comm_inst.offlinePeers \
and i not in gettransports.get() and len(str(i).strip()) > 0:
if not db_only:
peerList.append(i)
keydb.addkeys.add_address(i)

43
src/communicator/peeraction.py
View File

@ -1,9 +1,13 @@
'''
Onionr - Private P2P Communication
"""Onionr - Private P2P Communication.
This file implements logic for performing requests to Onionr peers
'''
'''
This file implements logic for performing requests to Onionr peers
"""
import streamedrequests
import logger
from onionrutils import epoch, basicrequests
from coredb import keydb
from . import onlinepeers
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@ -16,15 +20,12 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import streamedrequests
import logger
from onionrutils import epoch, basicrequests
from coredb import keydb
from . import onlinepeers
"""
def peer_action(comm_inst, peer, action, returnHeaders=False, max_resp_size=5242880):
'''Perform a get request to a peer'''
def peer_action(comm_inst, peer, action,
returnHeaders=False, max_resp_size=5242880):
"""Perform a get request to a peer."""
penalty_score = -10
if len(peer) == 0:
return False
@ -34,22 +35,28 @@ def peer_action(comm_inst, peer, action, returnHeaders=False, max_resp_size=5242
ret_data = basicrequests.do_get_request(url, port=comm_inst.proxyPort,
max_size=max_resp_size)
except streamedrequests.exceptions.ResponseLimitReached:
logger.warn('Request failed due to max response size being overflowed', terminal=True)
logger.warn(
'Request failed due to max response size being overflowed',
terminal=True)
ret_data = False
penalty_score = -100
# if request failed, (error), mark peer offline
if ret_data == False: # For some reason "if not" breaks this. Prob has to do with empty string.
if ret_data is False:
try:
comm_inst.getPeerProfileInstance(peer).addScore(penalty_score)
onlinepeers.remove_online_peer(comm_inst, peer)
keydb.transportinfo.set_address_info(peer, 'lastConnectAttempt', epoch.get_epoch())
keydb.transportinfo.set_address_info(
peer, 'lastConnectAttempt', epoch.get_epoch())
if action != 'ping' and not comm_inst.shutdown:
logger.warn(f'Lost connection to {peer}', terminal=True)
onlinepeers.get_online_peers(comm_inst) # Will only add a new peer to pool if needed
# Will only add a new peer to pool if needed
onlinepeers.get_online_peers(comm_inst)
except ValueError:
pass
else:
peer_profile = comm_inst.getPeerProfileInstance(peer)
peer_profile.update_connect_time()
peer_profile.addScore(1)
return ret_data # If returnHeaders, returns tuple of data, headers. if not, just data string
# If returnHeaders, returns tuple of data, headers.
# If not, just data string
return ret_data

66
src/communicatorutils/announcenode.py
View File

@ -1,9 +1,20 @@
'''
Onionr - Private P2P Communication
"""
Onionr - Private P2P Communication.
Use a communicator instance to announce our transport address to connected nodes
'''
'''
Use a communicator instance to announce
our transport address to connected nodes
"""
import base64
import onionrproofs
import logger
from etc import onionrvalues
from onionrutils import basicrequests, bytesconverter
from utils import gettransports
from netcontroller import NetController
from communicator import onlinepeers
from coredb import keydb
import onionrexceptions
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@ -16,22 +27,14 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import base64
import onionrproofs, logger
from etc import onionrvalues
from onionrutils import basicrequests, bytesconverter
from utils import gettransports
from netcontroller import NetController
from communicator import onlinepeers
from coredb import keydb
import onionrexceptions
"""
def announce_node(daemon):
'''Announce our node to our peers'''
"""Announce our node to our peers."""
ret_data = False
announce_fail = False
# Do not let announceCache get too large
if len(daemon.announceCache) >= 10000:
daemon.announceCache.popitem()
@ -39,7 +42,8 @@ def announce_node(daemon):
if daemon.config.get('general.security_level', 0) == 0:
# Announce to random online peers
for i in daemon.onlinePeers:
if not i in daemon.announceCache and not i in daemon.announceProgress:
if i not in daemon.announceCache and\
i not in daemon.announceProgress:
peer = i
break
else:
@ -61,9 +65,12 @@ def announce_node(daemon):
combinedNodes = ourID + peer
if ourID != 1:
existingRand = bytesconverter.bytes_to_str(keydb.transportinfo.get_address_info(peer, 'powValue'))
existingRand = bytesconverter.bytes_to_str(
keydb.transportinfo.get_address_info(peer, 'powValue'))
# Reset existingRand if it no longer meets the minimum POW
if type(existingRand) is type(None) or not existingRand.endswith('0' * onionrvalues.ANNOUNCE_POW):
if isinstance(existingRand, type(None)) or \
not existingRand.endswith(
'0' * onionrvalues.ANNOUNCE_POW):
existingRand = ''
if peer in daemon.announceCache:
@ -72,22 +79,29 @@ def announce_node(daemon):
data['random'] = existingRand
else:
daemon.announceProgress[peer] = True
proof = onionrproofs.DataPOW(combinedNodes, minDifficulty=onionrvalues.ANNOUNCE_POW)
proof = onionrproofs.DataPOW(
combinedNodes, minDifficulty=onionrvalues.ANNOUNCE_POW)
del daemon.announceProgress[peer]
try:
data['random'] = base64.b64encode(proof.waitForResult()[1])
except TypeError:
# Happens when we failed to produce a proof
logger.error("Failed to produce a pow for announcing to " + peer)
logger.error(f"Failed to produce a pow for {peer} annce")
announce_fail = True
else:
daemon.announceCache[peer] = data['random']
if not announce_fail:
logger.info('Announcing node to ' + url)
if basicrequests.do_post_request(url, data, port=daemon.shared_state.get(NetController).socksPort) == 'Success':
logger.info('Successfully introduced node to ' + peer, terminal=True)
if basicrequests.do_post_request(
url,
data,
port=daemon.shared_state.get(NetController).socksPort)\
== 'Success':
logger.info('Successfully introduced node to ' + peer,
terminal=True)
ret_data = True
keydb.transportinfo.set_address_info(peer, 'introduced', 1)
keydb.transportinfo.set_address_info(peer, 'powValue', data['random'])
keydb.transportinfo.set_address_info(peer, 'powValue',
data['random'])
daemon.decrementThreadCount('announce_node')
return ret_data
return ret_data

38
src/communicatorutils/downloadblocks/shoulddownload.py
View File

@ -1,9 +1,11 @@
'''
Onionr - Private P2P Communication
"""Onionr - Private P2P Communication.
Check if a block should be downloaded (if we already have it or its blacklisted or not)
'''
'''
Check if a block should be downloaded
(if we already have it or its blacklisted or not)
"""
from coredb import blockmetadb
from onionrblocks import onionrblacklist
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@ -16,22 +18,24 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
from coredb import blockmetadb
from onionrblocks import onionrblacklist
"""
def should_download(comm_inst, block_hash):
def should_download(comm_inst, block_hash) -> bool:
"""Return bool for if a (assumed to exist) block should be downloaded."""
blacklist = onionrblacklist.OnionrBlackList()
ret_data = True
if block_hash in blockmetadb.get_block_list(): # Dont download block we have
ret_data = False
should = True
if block_hash in blockmetadb.get_block_list():
# Don't download block we have
should = False
else:
if blacklist.inBlacklist(block_hash): # Dont download blacklisted block
ret_data = False
if ret_data is False:
# Remove block from communicator queue if it shouldnt be downloaded
if blacklist.inBlacklist(block_hash):
# Don't download blacklisted block
should = False
if should is False:
# Remove block from communicator queue if it shouldn't be downloaded
try:
del comm_inst.blockQueue[block_hash]
except KeyError:
pass
return ret_data
return should

74
src/communicatorutils/uploadblocks/__init__.py
View File

@ -1,10 +1,22 @@
'''
Onionr - Private P2P Communication
"""Onionr - Private P2P Communication.
Upload blocks in the upload queue to peers from the communicator
'''
from __future__ import annotations
'''
Upload blocks in the upload queue to peers from the communicator
"""
from typing import TYPE_CHECKING
from . import sessionmanager
from onionrtypes import UserID
import logger
from communicatorutils import proxypicker
import onionrexceptions
from onionrblocks import onionrblockapi as block
from onionrutils import stringvalidators, basicrequests
import onionrcrypto
from communicator import onlinepeers
if TYPE_CHECKING:
from communicator import OnionrCommunicatorDaemon
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@ -17,28 +29,23 @@ from __future__ import annotations
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
from typing import Union, TYPE_CHECKING
import logger
from communicatorutils import proxypicker
import onionrexceptions
from onionrblocks import onionrblockapi as block
from onionrutils import localcommand, stringvalidators, basicrequests
from communicator import onlinepeers
import onionrcrypto
from . import sessionmanager
from . import mixmate
"""
def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
"""Accepts a communicator instance and uploads blocks from its upload queue"""
def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'):
"""Accept a communicator instance + upload blocks from its upload queue."""
"""when inserting a block, we try to upload
it to a few peers to add some deniability & increase functionality"""
TIMER_NAME = "upload_blocks_from_communicator"
session_manager: sessionmanager.BlockUploadSessionManager = comm_inst.shared_state.get(sessionmanager.BlockUploadSessionManager)
triedPeers = []
session_manager: sessionmanager.BlockUploadSessionManager
session_manager = comm_inst.shared_state.get(
sessionmanager.BlockUploadSessionManager)
tried_peers: UserID = []
finishedUploads = []
comm_inst.blocksToUpload = onionrcrypto.cryptoutils.random_shuffle(comm_inst.blocksToUpload)
comm_inst.blocksToUpload = onionrcrypto.cryptoutils.random_shuffle(
comm_inst.blocksToUpload)
if len(comm_inst.blocksToUpload) != 0:
for bl in comm_inst.blocksToUpload:
if not stringvalidators.validate_hash(bl):
@ -57,21 +64,26 @@ def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
except KeyError:
pass
try:
if session.peer_fails[peer] > 3: continue
if session.peer_fails[peer] > 3:
continue
except KeyError:
pass
if peer in triedPeers: continue
triedPeers.append(peer)
if peer in tried_peers:
continue
tried_peers.append(peer)
url = f'http://{peer}/upload'
try:
data = block.Block(bl).getRaw()
except onionrexceptions.NoDataAvailable:
finishedUploads.append(bl)
break
proxyType = proxypicker.pick_proxy(peer)
logger.info(f"Uploading block {bl[:8]} to {peer}", terminal=True)
resp = basicrequests.do_post_request(url, data=data, proxyType=proxyType, content_type='application/octet-stream')
if not resp == False:
proxy_type = proxypicker.pick_proxy(peer)
logger.info(
f"Uploading block {bl[:8]} to {peer}", terminal=True)
resp = basicrequests.do_post_request(
url, data=data, proxyType=proxy_type,
content_type='application/octet-stream')
if resp is not False:
if resp == 'success':
session.success()
session.peer_exists[peer] = True
@ -82,7 +94,9 @@ def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
session.fail()
session.fail_peer(peer)
comm_inst.getPeerProfileInstance(peer).addScore(-5)
logger.warn(f'Failed to upload {bl[:8]}, reason: {resp[:15]}', terminal=True)
logger.warn(
f'Failed to upload {bl[:8]}, reason: {resp[:15]}',
terminal=True)
else:
session.fail()
session_manager.clean_session()

3
src/communicatorutils/uploadblocks/mixmate/pool.py
View File

@ -62,7 +62,8 @@ class UploadPool:
"""Get the hash pool in secure random order."""
if len(self._pool) != self._pool_size:
raise PoolNotReady
final_pool: List[onionrtypes.BlockHash] = cryptoutils.random_shuffle(list(self._pool))
final_pool: List[onionrtypes.BlockHash] = cryptoutils.random_shuffle(
list(self._pool))
self._pool.clear()
self.birthday = onionrutils.epoch.get_epoch()

7
src/communicatorutils/uploadblocks/session.py
View File

@ -2,8 +2,9 @@
Virtual upload "sessions" for blocks
"""
from typing import Union
from typing import Union, Dict
from onionrtypes import UserID
from onionrutils import stringvalidators
from onionrutils import bytesconverter
from onionrutils import epoch
@ -40,8 +41,8 @@ class UploadSession:
self.block_hash = reconstructhash.deconstruct_hash(block_hash)
self.total_fail_count: int = 0
self.total_success_count: int = 0
self.peer_fails = {}
self.peer_exists = {}
self.peer_fails: Dict[UserID, int] = {}
self.peer_exists: Dict[UserID, bool] = {}
def fail_peer(self, peer):
try:

119
src/communicatorutils/uploadblocks/sessionmanager.py
View File

@ -1,9 +1,17 @@
"""
Onionr - Private P2P Communication
"""Onionr - Private P2P Communication.
Manager for upload 'sessions'
Manager for upload 'sessions'
"""
from __future__ import annotations
from typing import List, Union, TYPE_CHECKING
if TYPE_CHECKING:
from session import UploadSession
from onionrutils import bytesconverter
from onionrutils import localcommand
from etc import onionrvalues
from utils import reconstructhash
from . import session
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -18,31 +26,33 @@ from __future__ import annotations
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
from typing import Iterable, Union
from onionrutils import bytesconverter
from onionrutils import localcommand
from etc import onionrvalues
from etc import waitforsetvar
from utils import reconstructhash
from . import session
class BlockUploadSessionManager:
"""Holds block UploadSession instances. Optionally accepts iterable of sessions to added on init
"""Holds block UploadSession instances.
Arguments: old_session: iterable of old UploadSession objects"""
def __init__(self, old_sessions:Iterable=None):
#self._too_many: TooMany = None
Optionally accepts iterable of sessions to added on init
Arguments: old_session: iterable of old UploadSession objects
"""
def __init__(self, old_sessions: List = None):
if old_sessions is None:
self.sessions = []
else:
self.sessions = old_session
def add_session(self, session_or_block: Union(str, bytes, session.UploadSession))->session.UploadSession:
"""Create (or add existing) block upload session from a str/bytes block hex hash, existing UploadSession"""
if isinstance(session_or_block, session.UploadSession):
if not session_or_block in self.sessions:
self.sessions = old_sessions
def add_session(self,
session_or_block: Union[str,
bytes,
session.UploadSession
]
) -> session.UploadSession:
"""Create (or add existing) block upload session.
from a str/bytes block hex hash, existing UploadSession
"""
if isinstance(session_or_block, session.UploadSession):
if session_or_block not in self.sessions:
self.sessions.append(session_or_block)
return session_or_block
try:
@ -50,46 +60,63 @@ class BlockUploadSessionManager:
except KeyError:
pass
# convert bytes hash to str
if isinstance(session_or_block, bytes): session_or_block = bytesconverter.bytes_to_str(session_or_block)
if isinstance(session_or_block, bytes):
session_or_block = bytesconverter.bytes_to_str(session_or_block)
# intentionally not elif
if isinstance(session_or_block, str):
new_session = session.UploadSession(session_or_block)
self.sessions.append(new_session)
return new_session
raise ValueError
def get_session(self, block_hash: Union(str, bytes))->session.UploadSession:
block_hash = reconstructhash.deconstruct_hash(bytesconverter.bytes_to_str(block_hash))
for session in self.sessions:
if session.block_hash == block_hash: return session
def get_session(self,
block_hash: Union[str, bytes]
) -> session.UploadSession:
block_hash = reconstructhash.deconstruct_hash(
bytesconverter.bytes_to_str(block_hash))
for sess in self.sessions:
if sess.block_hash == block_hash:
return sess
raise KeyError
def clean_session(self, specific_session: Union[str, UploadSession]=None):
comm_inst: OnionrCommunicatorDaemon = self._too_many.get_by_string("OnionrCommunicatorDaemon")
sessions_to_delete = []
if comm_inst.getUptime() < 120: return
onlinePeerCount = len(comm_inst.onlinePeers)
# If we have no online peers right now,
if onlinePeerCount == 0: return
def clean_session(self,
specific_session: Union[str, 'UploadSession'] = None):
for session in self.sessions:
# if over 50% of peers that were online for a session have become unavailable, don't kill sessions
if session.total_success_count > onlinePeerCount:
if onlinePeerCount / session.total_success_count >= 0.5: return
comm_inst: 'OnionrCommunicatorDaemon' # type: ignore
comm_inst = self._too_many.get_by_string( # pylint: disable=E1101 type: ignore
"OnionrCommunicatorDaemon")
sessions_to_delete = []
if comm_inst.getUptime() < 120:
return
onlinePeerCount = len(comm_inst.onlinePeers)
# If we have no online peers right now,
if onlinePeerCount == 0:
return
for sess in self.sessions:
# if over 50% of peers that were online for a session have
# become unavailable, don't kill sessions
if sess.total_success_count > onlinePeerCount:
if onlinePeerCount / sess.total_success_count >= 0.5:
return
# Clean sessions if they have uploaded to enough online peers
if session.total_success_count <= 0: continue
if (session.total_success_count / onlinePeerCount) >= onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT:
sessions_to_delete.append(session)
for session in sessions_to_delete:
if sess.total_success_count <= 0:
continue
if (sess.total_success_count / onlinePeerCount) >= onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT:
sessions_to_delete.append(sess)
for sess in sessions_to_delete:
self.sessions.remove(session)
# TODO cleanup to one round of search
# Remove the blocks from the sessions, upload list, and waitforshare list
# Remove the blocks from the sessions, upload list,
# and waitforshare list
try:
comm_inst.blocksToUpload.remove(reconstructhash.reconstruct_hash(session.block_hash))
comm_inst.blocksToUpload.remove(
reconstructhash.reconstruct_hash(sess.block_hash))
except ValueError:
pass
try:
comm_inst.blocksToUpload.remove(session.block_hash)
comm_inst.blocksToUpload.remove(sess.block_hash)
except ValueError:
pass
localcommand.local_command('waitforshare/{session.block_hash}')