diff --git a/src/communicator/bootstrappeers.py b/src/communicator/bootstrappeers.py
index dab2b3f9..b76f4546 100644
--- a/src/communicator/bootstrappeers.py
+++ b/src/communicator/bootstrappeers.py
@@ -1,9 +1,10 @@
-'''
- Onionr - Private P2P Communication
+"""Onionr - Private P2P Communication.
- add bootstrap peers to the communicator peer list
-'''
-'''
+add bootstrap peers to the communicator peer list
+"""
+from utils import readstatic, gettransports
+from coredb import keydb
+"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@@ -16,16 +17,16 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see .
-'''
-from utils import readstatic, gettransports
-from coredb import keydb
+"""
+
bootstrap_peers = readstatic.read_static('bootstrap-nodes.txt').split(',')
+
def add_bootstrap_list_to_peer_list(comm_inst, peerList, db_only=False):
- '''
- Add the bootstrap list to the peer list (no duplicates)
- '''
+ """Add the bootstrap list to the peer list (no duplicates)."""
for i in bootstrap_peers:
- if i not in peerList and i not in comm_inst.offlinePeers and not i in gettransports.get() and len(str(i).strip()) > 0:
- if not db_only: peerList.append(i)
+ if i not in peerList and i not in comm_inst.offlinePeers \
+ and i not in gettransports.get() and len(str(i).strip()) > 0:
+ if not db_only:
+ peerList.append(i)
keydb.addkeys.add_address(i)
diff --git a/src/communicator/peeraction.py b/src/communicator/peeraction.py
index ca715273..aeda75c1 100644
--- a/src/communicator/peeraction.py
+++ b/src/communicator/peeraction.py
@@ -1,9 +1,13 @@
-'''
- Onionr - Private P2P Communication
+"""Onionr - Private P2P Communication.
- This file implements logic for performing requests to Onionr peers
-'''
-'''
+This file implements logic for performing requests to Onionr peers
+"""
+import streamedrequests
+import logger
+from onionrutils import epoch, basicrequests
+from coredb import keydb
+from . import onlinepeers
+"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@@ -16,15 +20,12 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see .
-'''
-import streamedrequests
-import logger
-from onionrutils import epoch, basicrequests
-from coredb import keydb
-from . import onlinepeers
+"""
-def peer_action(comm_inst, peer, action, returnHeaders=False, max_resp_size=5242880):
- '''Perform a get request to a peer'''
+
+def peer_action(comm_inst, peer, action,
+ returnHeaders=False, max_resp_size=5242880):
+ """Perform a get request to a peer."""
penalty_score = -10
if len(peer) == 0:
return False
@@ -34,22 +35,28 @@ def peer_action(comm_inst, peer, action, returnHeaders=False, max_resp_size=5242
ret_data = basicrequests.do_get_request(url, port=comm_inst.proxyPort,
max_size=max_resp_size)
except streamedrequests.exceptions.ResponseLimitReached:
- logger.warn('Request failed due to max response size being overflowed', terminal=True)
+ logger.warn(
+ 'Request failed due to max response size being overflowed',
+ terminal=True)
ret_data = False
penalty_score = -100
# if request failed, (error), mark peer offline
- if ret_data == False: # For some reason "if not" breaks this. Prob has to do with empty string.
+ if ret_data is False:
try:
comm_inst.getPeerProfileInstance(peer).addScore(penalty_score)
onlinepeers.remove_online_peer(comm_inst, peer)
- keydb.transportinfo.set_address_info(peer, 'lastConnectAttempt', epoch.get_epoch())
+ keydb.transportinfo.set_address_info(
+ peer, 'lastConnectAttempt', epoch.get_epoch())
if action != 'ping' and not comm_inst.shutdown:
logger.warn(f'Lost connection to {peer}', terminal=True)
- onlinepeers.get_online_peers(comm_inst) # Will only add a new peer to pool if needed
+ # Will only add a new peer to pool if needed
+ onlinepeers.get_online_peers(comm_inst)
except ValueError:
pass
else:
peer_profile = comm_inst.getPeerProfileInstance(peer)
peer_profile.update_connect_time()
peer_profile.addScore(1)
- return ret_data # If returnHeaders, returns tuple of data, headers. if not, just data string
+ # If returnHeaders, returns tuple of data, headers.
+ # If not, just data string
+ return ret_data
diff --git a/src/communicatorutils/announcenode.py b/src/communicatorutils/announcenode.py
index 4667afc7..1d92a3fa 100755
--- a/src/communicatorutils/announcenode.py
+++ b/src/communicatorutils/announcenode.py
@@ -1,9 +1,20 @@
-'''
- Onionr - Private P2P Communication
+"""
+ Onionr - Private P2P Communication.
- Use a communicator instance to announce our transport address to connected nodes
-'''
-'''
+Use a communicator instance to announce
+our transport address to connected nodes
+"""
+import base64
+import onionrproofs
+import logger
+from etc import onionrvalues
+from onionrutils import basicrequests, bytesconverter
+from utils import gettransports
+from netcontroller import NetController
+from communicator import onlinepeers
+from coredb import keydb
+import onionrexceptions
+"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@@ -16,22 +27,14 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see .
-'''
-import base64
-import onionrproofs, logger
-from etc import onionrvalues
-from onionrutils import basicrequests, bytesconverter
-from utils import gettransports
-from netcontroller import NetController
-from communicator import onlinepeers
-from coredb import keydb
-import onionrexceptions
+"""
+
def announce_node(daemon):
- '''Announce our node to our peers'''
+ """Announce our node to our peers."""
ret_data = False
announce_fail = False
-
+
# Do not let announceCache get too large
if len(daemon.announceCache) >= 10000:
daemon.announceCache.popitem()
@@ -39,7 +42,8 @@ def announce_node(daemon):
if daemon.config.get('general.security_level', 0) == 0:
# Announce to random online peers
for i in daemon.onlinePeers:
- if not i in daemon.announceCache and not i in daemon.announceProgress:
+ if i not in daemon.announceCache and\
+ i not in daemon.announceProgress:
peer = i
break
else:
@@ -61,9 +65,12 @@ def announce_node(daemon):
combinedNodes = ourID + peer
if ourID != 1:
- existingRand = bytesconverter.bytes_to_str(keydb.transportinfo.get_address_info(peer, 'powValue'))
+ existingRand = bytesconverter.bytes_to_str(
+ keydb.transportinfo.get_address_info(peer, 'powValue'))
# Reset existingRand if it no longer meets the minimum POW
- if type(existingRand) is type(None) or not existingRand.endswith('0' * onionrvalues.ANNOUNCE_POW):
+ if isinstance(existingRand, type(None)) or \
+ not existingRand.endswith(
+ '0' * onionrvalues.ANNOUNCE_POW):
existingRand = ''
if peer in daemon.announceCache:
@@ -72,22 +79,29 @@ def announce_node(daemon):
data['random'] = existingRand
else:
daemon.announceProgress[peer] = True
- proof = onionrproofs.DataPOW(combinedNodes, minDifficulty=onionrvalues.ANNOUNCE_POW)
+ proof = onionrproofs.DataPOW(
+ combinedNodes, minDifficulty=onionrvalues.ANNOUNCE_POW)
del daemon.announceProgress[peer]
try:
data['random'] = base64.b64encode(proof.waitForResult()[1])
except TypeError:
# Happens when we failed to produce a proof
- logger.error("Failed to produce a pow for announcing to " + peer)
+ logger.error(f"Failed to produce a pow for {peer} annce")
announce_fail = True
else:
daemon.announceCache[peer] = data['random']
if not announce_fail:
logger.info('Announcing node to ' + url)
- if basicrequests.do_post_request(url, data, port=daemon.shared_state.get(NetController).socksPort) == 'Success':
- logger.info('Successfully introduced node to ' + peer, terminal=True)
+ if basicrequests.do_post_request(
+ url,
+ data,
+ port=daemon.shared_state.get(NetController).socksPort)\
+ == 'Success':
+ logger.info('Successfully introduced node to ' + peer,
+ terminal=True)
ret_data = True
keydb.transportinfo.set_address_info(peer, 'introduced', 1)
- keydb.transportinfo.set_address_info(peer, 'powValue', data['random'])
+ keydb.transportinfo.set_address_info(peer, 'powValue',
+ data['random'])
daemon.decrementThreadCount('announce_node')
- return ret_data
\ No newline at end of file
+ return ret_data
diff --git a/src/communicatorutils/downloadblocks/shoulddownload.py b/src/communicatorutils/downloadblocks/shoulddownload.py
index b59ed47f..2e296eb0 100644
--- a/src/communicatorutils/downloadblocks/shoulddownload.py
+++ b/src/communicatorutils/downloadblocks/shoulddownload.py
@@ -1,9 +1,11 @@
-'''
- Onionr - Private P2P Communication
+"""Onionr - Private P2P Communication.
- Check if a block should be downloaded (if we already have it or its blacklisted or not)
-'''
-'''
+Check if a block should be downloaded
+(if we already have it or its blacklisted or not)
+"""
+from coredb import blockmetadb
+from onionrblocks import onionrblacklist
+"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@@ -16,22 +18,24 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see .
-'''
-from coredb import blockmetadb
-from onionrblocks import onionrblacklist
+"""
-def should_download(comm_inst, block_hash):
+
+def should_download(comm_inst, block_hash) -> bool:
+ """Return bool for if a (assumed to exist) block should be downloaded."""
blacklist = onionrblacklist.OnionrBlackList()
- ret_data = True
- if block_hash in blockmetadb.get_block_list(): # Dont download block we have
- ret_data = False
+ should = True
+ if block_hash in blockmetadb.get_block_list():
+ # Don't download block we have
+ should = False
else:
- if blacklist.inBlacklist(block_hash): # Dont download blacklisted block
- ret_data = False
- if ret_data is False:
- # Remove block from communicator queue if it shouldnt be downloaded
+ if blacklist.inBlacklist(block_hash):
+ # Don't download blacklisted block
+ should = False
+ if should is False:
+ # Remove block from communicator queue if it shouldn't be downloaded
try:
del comm_inst.blockQueue[block_hash]
except KeyError:
pass
- return ret_data
\ No newline at end of file
+ return should
diff --git a/src/communicatorutils/uploadblocks/__init__.py b/src/communicatorutils/uploadblocks/__init__.py
index b1420925..d0a53108 100755
--- a/src/communicatorutils/uploadblocks/__init__.py
+++ b/src/communicatorutils/uploadblocks/__init__.py
@@ -1,10 +1,22 @@
-'''
- Onionr - Private P2P Communication
+"""Onionr - Private P2P Communication.
- Upload blocks in the upload queue to peers from the communicator
-'''
-from __future__ import annotations
-'''
+Upload blocks in the upload queue to peers from the communicator
+"""
+from typing import TYPE_CHECKING
+from . import sessionmanager
+
+from onionrtypes import UserID
+import logger
+from communicatorutils import proxypicker
+import onionrexceptions
+from onionrblocks import onionrblockapi as block
+from onionrutils import stringvalidators, basicrequests
+import onionrcrypto
+from communicator import onlinepeers
+if TYPE_CHECKING:
+ from communicator import OnionrCommunicatorDaemon
+
+"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@@ -17,28 +29,23 @@ from __future__ import annotations
You should have received a copy of the GNU General Public License
along with this program. If not, see .
-'''
-from typing import Union, TYPE_CHECKING
-import logger
-from communicatorutils import proxypicker
-import onionrexceptions
-from onionrblocks import onionrblockapi as block
-from onionrutils import localcommand, stringvalidators, basicrequests
-from communicator import onlinepeers
-import onionrcrypto
-from . import sessionmanager
-from . import mixmate
+"""
-def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
- """Accepts a communicator instance and uploads blocks from its upload queue"""
+
+def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'):
+ """Accept a communicator instance + upload blocks from its upload queue."""
"""when inserting a block, we try to upload
it to a few peers to add some deniability & increase functionality"""
TIMER_NAME = "upload_blocks_from_communicator"
- session_manager: sessionmanager.BlockUploadSessionManager = comm_inst.shared_state.get(sessionmanager.BlockUploadSessionManager)
- triedPeers = []
+ session_manager: sessionmanager.BlockUploadSessionManager
+ session_manager = comm_inst.shared_state.get(
+ sessionmanager.BlockUploadSessionManager)
+ tried_peers: UserID = []
finishedUploads = []
- comm_inst.blocksToUpload = onionrcrypto.cryptoutils.random_shuffle(comm_inst.blocksToUpload)
+ comm_inst.blocksToUpload = onionrcrypto.cryptoutils.random_shuffle(
+ comm_inst.blocksToUpload)
+
if len(comm_inst.blocksToUpload) != 0:
for bl in comm_inst.blocksToUpload:
if not stringvalidators.validate_hash(bl):
@@ -57,21 +64,26 @@ def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
except KeyError:
pass
try:
- if session.peer_fails[peer] > 3: continue
+ if session.peer_fails[peer] > 3:
+ continue
except KeyError:
pass
- if peer in triedPeers: continue
- triedPeers.append(peer)
+ if peer in tried_peers:
+ continue
+ tried_peers.append(peer)
url = f'http://{peer}/upload'
try:
data = block.Block(bl).getRaw()
except onionrexceptions.NoDataAvailable:
finishedUploads.append(bl)
break
- proxyType = proxypicker.pick_proxy(peer)
- logger.info(f"Uploading block {bl[:8]} to {peer}", terminal=True)
- resp = basicrequests.do_post_request(url, data=data, proxyType=proxyType, content_type='application/octet-stream')
- if not resp == False:
+ proxy_type = proxypicker.pick_proxy(peer)
+ logger.info(
+ f"Uploading block {bl[:8]} to {peer}", terminal=True)
+ resp = basicrequests.do_post_request(
+ url, data=data, proxyType=proxy_type,
+ content_type='application/octet-stream')
+ if resp is not False:
if resp == 'success':
session.success()
session.peer_exists[peer] = True
@@ -82,7 +94,9 @@ def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
session.fail()
session.fail_peer(peer)
comm_inst.getPeerProfileInstance(peer).addScore(-5)
- logger.warn(f'Failed to upload {bl[:8]}, reason: {resp[:15]}', terminal=True)
+ logger.warn(
+ f'Failed to upload {bl[:8]}, reason: {resp[:15]}',
+ terminal=True)
else:
session.fail()
session_manager.clean_session()
diff --git a/src/communicatorutils/uploadblocks/mixmate/pool.py b/src/communicatorutils/uploadblocks/mixmate/pool.py
index 8cc944c1..87e98747 100644
--- a/src/communicatorutils/uploadblocks/mixmate/pool.py
+++ b/src/communicatorutils/uploadblocks/mixmate/pool.py
@@ -62,7 +62,8 @@ class UploadPool:
"""Get the hash pool in secure random order."""
if len(self._pool) != self._pool_size:
raise PoolNotReady
- final_pool: List[onionrtypes.BlockHash] = cryptoutils.random_shuffle(list(self._pool))
+ final_pool: List[onionrtypes.BlockHash] = cryptoutils.random_shuffle(
+ list(self._pool))
self._pool.clear()
self.birthday = onionrutils.epoch.get_epoch()
diff --git a/src/communicatorutils/uploadblocks/session.py b/src/communicatorutils/uploadblocks/session.py
index c3fdc2a9..d9b715bf 100644
--- a/src/communicatorutils/uploadblocks/session.py
+++ b/src/communicatorutils/uploadblocks/session.py
@@ -2,8 +2,9 @@
Virtual upload "sessions" for blocks
"""
-from typing import Union
+from typing import Union, Dict
+from onionrtypes import UserID
from onionrutils import stringvalidators
from onionrutils import bytesconverter
from onionrutils import epoch
@@ -40,8 +41,8 @@ class UploadSession:
self.block_hash = reconstructhash.deconstruct_hash(block_hash)
self.total_fail_count: int = 0
self.total_success_count: int = 0
- self.peer_fails = {}
- self.peer_exists = {}
+ self.peer_fails: Dict[UserID, int] = {}
+ self.peer_exists: Dict[UserID, bool] = {}
def fail_peer(self, peer):
try:
diff --git a/src/communicatorutils/uploadblocks/sessionmanager.py b/src/communicatorutils/uploadblocks/sessionmanager.py
index 1282cb79..81faa668 100644
--- a/src/communicatorutils/uploadblocks/sessionmanager.py
+++ b/src/communicatorutils/uploadblocks/sessionmanager.py
@@ -1,9 +1,17 @@
-"""
- Onionr - Private P2P Communication
+"""Onionr - Private P2P Communication.
- Manager for upload 'sessions'
+Manager for upload 'sessions'
"""
-from __future__ import annotations
+from typing import List, Union, TYPE_CHECKING
+if TYPE_CHECKING:
+ from session import UploadSession
+
+from onionrutils import bytesconverter
+from onionrutils import localcommand
+from etc import onionrvalues
+from utils import reconstructhash
+
+from . import session
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -18,31 +26,33 @@ from __future__ import annotations
You should have received a copy of the GNU General Public License
along with this program. If not, see .
"""
-from typing import Iterable, Union
-from onionrutils import bytesconverter
-from onionrutils import localcommand
-from etc import onionrvalues
-from etc import waitforsetvar
-from utils import reconstructhash
-
-from . import session
class BlockUploadSessionManager:
- """Holds block UploadSession instances. Optionally accepts iterable of sessions to added on init
+ """Holds block UploadSession instances.
- Arguments: old_session: iterable of old UploadSession objects"""
- def __init__(self, old_sessions:Iterable=None):
- #self._too_many: TooMany = None
+ Optionally accepts iterable of sessions to added on init
+ Arguments: old_session: iterable of old UploadSession objects
+ """
+
+ def __init__(self, old_sessions: List = None):
if old_sessions is None:
self.sessions = []
else:
- self.sessions = old_session
-
- def add_session(self, session_or_block: Union(str, bytes, session.UploadSession))->session.UploadSession:
- """Create (or add existing) block upload session from a str/bytes block hex hash, existing UploadSession"""
- if isinstance(session_or_block, session.UploadSession):
- if not session_or_block in self.sessions:
+ self.sessions = old_sessions
+
+ def add_session(self,
+ session_or_block: Union[str,
+ bytes,
+ session.UploadSession
+ ]
+ ) -> session.UploadSession:
+ """Create (or add existing) block upload session.
+
+ from a str/bytes block hex hash, existing UploadSession
+ """
+ if isinstance(session_or_block, session.UploadSession):
+ if session_or_block not in self.sessions:
self.sessions.append(session_or_block)
return session_or_block
try:
@@ -50,46 +60,63 @@ class BlockUploadSessionManager:
except KeyError:
pass
# convert bytes hash to str
- if isinstance(session_or_block, bytes): session_or_block = bytesconverter.bytes_to_str(session_or_block)
+ if isinstance(session_or_block, bytes):
+ session_or_block = bytesconverter.bytes_to_str(session_or_block)
# intentionally not elif
if isinstance(session_or_block, str):
new_session = session.UploadSession(session_or_block)
self.sessions.append(new_session)
return new_session
+ raise ValueError
- def get_session(self, block_hash: Union(str, bytes))->session.UploadSession:
- block_hash = reconstructhash.deconstruct_hash(bytesconverter.bytes_to_str(block_hash))
- for session in self.sessions:
- if session.block_hash == block_hash: return session
+ def get_session(self,
+ block_hash: Union[str, bytes]
+ ) -> session.UploadSession:
+ block_hash = reconstructhash.deconstruct_hash(
+ bytesconverter.bytes_to_str(block_hash))
+ for sess in self.sessions:
+ if sess.block_hash == block_hash:
+ return sess
raise KeyError
- def clean_session(self, specific_session: Union[str, UploadSession]=None):
- comm_inst: OnionrCommunicatorDaemon = self._too_many.get_by_string("OnionrCommunicatorDaemon")
- sessions_to_delete = []
- if comm_inst.getUptime() < 120: return
- onlinePeerCount = len(comm_inst.onlinePeers)
-
- # If we have no online peers right now,
- if onlinePeerCount == 0: return
+ def clean_session(self,
+ specific_session: Union[str, 'UploadSession'] = None):
- for session in self.sessions:
- # if over 50% of peers that were online for a session have become unavailable, don't kill sessions
- if session.total_success_count > onlinePeerCount:
- if onlinePeerCount / session.total_success_count >= 0.5: return
+ comm_inst: 'OnionrCommunicatorDaemon' # type: ignore
+ comm_inst = self._too_many.get_by_string( # pylint: disable=E1101 type: ignore
+ "OnionrCommunicatorDaemon")
+ sessions_to_delete = []
+ if comm_inst.getUptime() < 120:
+ return
+ onlinePeerCount = len(comm_inst.onlinePeers)
+
+ # If we have no online peers right now,
+ if onlinePeerCount == 0:
+ return
+
+ for sess in self.sessions:
+ # if over 50% of peers that were online for a session have
+ # become unavailable, don't kill sessions
+ if sess.total_success_count > onlinePeerCount:
+ if onlinePeerCount / sess.total_success_count >= 0.5:
+ return
# Clean sessions if they have uploaded to enough online peers
- if session.total_success_count <= 0: continue
- if (session.total_success_count / onlinePeerCount) >= onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT:
- sessions_to_delete.append(session)
- for session in sessions_to_delete:
+ if sess.total_success_count <= 0:
+ continue
+ if (sess.total_success_count / onlinePeerCount) >= onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT:
+ sessions_to_delete.append(sess)
+ for sess in sessions_to_delete:
self.sessions.remove(session)
# TODO cleanup to one round of search
- # Remove the blocks from the sessions, upload list, and waitforshare list
+ # Remove the blocks from the sessions, upload list,
+ # and waitforshare list
try:
- comm_inst.blocksToUpload.remove(reconstructhash.reconstruct_hash(session.block_hash))
+ comm_inst.blocksToUpload.remove(
+ reconstructhash.reconstruct_hash(sess.block_hash))
except ValueError:
pass
try:
- comm_inst.blocksToUpload.remove(session.block_hash)
+ comm_inst.blocksToUpload.remove(sess.block_hash)
except ValueError:
pass
localcommand.local_command('waitforshare/{session.block_hash}')