minor bug fixes and linting improvements

src/communicator/bootstrappeers.py

@@ -1,9 +1,10 @@
-'''
+"""Onionr - Private P2P Communication.
-    Onionr - Private P2P Communication
 
-    add bootstrap peers to the communicator peer list
+add bootstrap peers to the communicator peer list
-'''
+"""
-'''
+from utils import readstatic, gettransports
+from coredb import keydb
+"""
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation, either version 3 of the License, or
@@ -16,16 +17,16 @@
 
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
+"""
-from utils import readstatic, gettransports
+
-from coredb import keydb
 bootstrap_peers = readstatic.read_static('bootstrap-nodes.txt').split(',')
 
+
 def add_bootstrap_list_to_peer_list(comm_inst, peerList, db_only=False):
-    '''
+    """Add the bootstrap list to the peer list (no duplicates)."""
-        Add the bootstrap list to the peer list (no duplicates)
-    '''
     for i in bootstrap_peers:
-        if i not in peerList and i not in comm_inst.offlinePeers and not i in gettransports.get() and len(str(i).strip()) > 0:
+        if i not in peerList and i not in comm_inst.offlinePeers \
-            if not db_only: peerList.append(i)
+                and i not in gettransports.get() and len(str(i).strip()) > 0:
+            if not db_only:
+                peerList.append(i)
             keydb.addkeys.add_address(i)

src/communicator/peeraction.py

@@ -1,9 +1,13 @@
-'''
+"""Onionr - Private P2P Communication.
-    Onionr - Private P2P Communication
 
-    This file implements logic for performing requests to Onionr peers
+This file implements logic for performing requests to Onionr peers
-'''
+"""
-'''
+import streamedrequests
+import logger
+from onionrutils import epoch, basicrequests
+from coredb import keydb
+from . import onlinepeers
+"""
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation, either version 3 of the License, or
@@ -16,15 +20,12 @@
 
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
+"""
-import streamedrequests
-import logger
-from onionrutils import epoch, basicrequests
-from coredb import keydb
-from . import onlinepeers
 
-def peer_action(comm_inst, peer, action, returnHeaders=False, max_resp_size=5242880):
+
-    '''Perform a get request to a peer'''
+def peer_action(comm_inst, peer, action,
+                returnHeaders=False, max_resp_size=5242880):
+    """Perform a get request to a peer."""
     penalty_score = -10
     if len(peer) == 0:
         return False
@@ -34,22 +35,28 @@ def peer_action(comm_inst, peer, action, returnHeaders=False, max_resp_size=5242
         ret_data = basicrequests.do_get_request(url, port=comm_inst.proxyPort,
                                                 max_size=max_resp_size)
     except streamedrequests.exceptions.ResponseLimitReached:
-        logger.warn('Request failed due to max response size being overflowed', terminal=True)
+        logger.warn(
+            'Request failed due to max response size being overflowed',
+            terminal=True)
         ret_data = False
         penalty_score = -100
     # if request failed, (error), mark peer offline
-    if ret_data == False: # For some reason "if not" breaks this. Prob has to do with empty string.
+    if ret_data is False:
         try:
             comm_inst.getPeerProfileInstance(peer).addScore(penalty_score)
             onlinepeers.remove_online_peer(comm_inst, peer)
-            keydb.transportinfo.set_address_info(peer, 'lastConnectAttempt', epoch.get_epoch())
+            keydb.transportinfo.set_address_info(
+                peer, 'lastConnectAttempt', epoch.get_epoch())
             if action != 'ping' and not comm_inst.shutdown:
                 logger.warn(f'Lost connection to {peer}', terminal=True)
-                onlinepeers.get_online_peers(comm_inst) # Will only add a new peer to pool if needed
+                # Will only add a new peer to pool if needed
+                onlinepeers.get_online_peers(comm_inst)
         except ValueError:
             pass
     else:
         peer_profile = comm_inst.getPeerProfileInstance(peer)
         peer_profile.update_connect_time()
         peer_profile.addScore(1)
-    return ret_data # If returnHeaders, returns tuple of data, headers. if not, just data string
+    # If returnHeaders, returns tuple of data, headers.
+    # If not, just data string
+    return ret_data

src/communicatorutils/announcenode.py

@@ -1,9 +1,20 @@
-'''
+"""
-    Onionr - Private P2P Communication
+    Onionr - Private P2P Communication.
 
-    Use a communicator instance to announce our transport address to connected nodes
+Use a communicator instance to announce
-'''
+our transport address to connected nodes
-'''
+"""
+import base64
+import onionrproofs
+import logger
+from etc import onionrvalues
+from onionrutils import basicrequests, bytesconverter
+from utils import gettransports
+from netcontroller import NetController
+from communicator import onlinepeers
+from coredb import keydb
+import onionrexceptions
+"""
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation, either version 3 of the License, or
@@ -16,22 +27,14 @@
 
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
+"""
-import base64
+
-import onionrproofs, logger
-from etc import onionrvalues
-from onionrutils import basicrequests, bytesconverter
-from utils import gettransports
-from netcontroller import NetController
-from communicator import onlinepeers
-from coredb import keydb
-import onionrexceptions
 
 def announce_node(daemon):
-    '''Announce our node to our peers'''
+    """Announce our node to our peers."""
     ret_data = False
     announce_fail = False
-    
+
     # Do not let announceCache get too large
     if len(daemon.announceCache) >= 10000:
         daemon.announceCache.popitem()
@@ -39,7 +42,8 @@ def announce_node(daemon):
     if daemon.config.get('general.security_level', 0) == 0:
         # Announce to random online peers
         for i in daemon.onlinePeers:
-            if not i in daemon.announceCache and not i in daemon.announceProgress:
+            if i not in daemon.announceCache and\
+                    i not in daemon.announceProgress:
                 peer = i
                 break
         else:
@@ -61,9 +65,12 @@ def announce_node(daemon):
 
             combinedNodes = ourID + peer
             if ourID != 1:
-                existingRand = bytesconverter.bytes_to_str(keydb.transportinfo.get_address_info(peer, 'powValue'))
+                existingRand = bytesconverter.bytes_to_str(
+                    keydb.transportinfo.get_address_info(peer, 'powValue'))
                 # Reset existingRand if it no longer meets the minimum POW
-                if type(existingRand) is type(None) or not existingRand.endswith('0' * onionrvalues.ANNOUNCE_POW):
+                if isinstance(existingRand, type(None)) or \
+                    not existingRand.endswith(
+                        '0' * onionrvalues.ANNOUNCE_POW):
                     existingRand = ''
 
             if peer in daemon.announceCache:
@@ -72,22 +79,29 @@ def announce_node(daemon):
                 data['random'] = existingRand
             else:
                 daemon.announceProgress[peer] = True
-                proof = onionrproofs.DataPOW(combinedNodes, minDifficulty=onionrvalues.ANNOUNCE_POW)
+                proof = onionrproofs.DataPOW(
+                    combinedNodes, minDifficulty=onionrvalues.ANNOUNCE_POW)
                 del daemon.announceProgress[peer]
                 try:
                     data['random'] = base64.b64encode(proof.waitForResult()[1])
                 except TypeError:
                     # Happens when we failed to produce a proof
-                    logger.error("Failed to produce a pow for announcing to " + peer)
+                    logger.error(f"Failed to produce a pow for {peer} annce")
                     announce_fail = True
                 else:
                     daemon.announceCache[peer] = data['random']
             if not announce_fail:
                 logger.info('Announcing node to ' + url)
-                if basicrequests.do_post_request(url, data, port=daemon.shared_state.get(NetController).socksPort) == 'Success':
+                if basicrequests.do_post_request(
-                    logger.info('Successfully introduced node to ' + peer, terminal=True)
+                        url,
+                        data,
+                        port=daemon.shared_state.get(NetController).socksPort)\
+                        == 'Success':
+                    logger.info('Successfully introduced node to ' + peer,
+                                terminal=True)
                     ret_data = True
                     keydb.transportinfo.set_address_info(peer, 'introduced', 1)
-                    keydb.transportinfo.set_address_info(peer, 'powValue', data['random'])
+                    keydb.transportinfo.set_address_info(peer, 'powValue',
+                                                         data['random'])
     daemon.decrementThreadCount('announce_node')
-    return ret_data
+    return ret_data

src/communicatorutils/downloadblocks/shoulddownload.py

@@ -1,9 +1,11 @@
-'''
+"""Onionr - Private P2P Communication.
-    Onionr - Private P2P Communication
 
-    Check if a block should be downloaded (if we already have it or its blacklisted or not)
+Check if a block should be downloaded
-'''
+(if we already have it or its blacklisted or not)
-'''
+"""
+from coredb import blockmetadb
+from onionrblocks import onionrblacklist
+"""
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation, either version 3 of the License, or
@@ -16,22 +18,24 @@
 
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
+"""
-from coredb import blockmetadb
-from onionrblocks import onionrblacklist
 
-def should_download(comm_inst, block_hash):
+
+def should_download(comm_inst, block_hash) -> bool:
+    """Return bool for if a (assumed to exist) block should be downloaded."""
     blacklist = onionrblacklist.OnionrBlackList()
-    ret_data = True
+    should = True
-    if block_hash in blockmetadb.get_block_list(): # Dont download block we have
+    if block_hash in blockmetadb.get_block_list():
-        ret_data = False
+        # Don't download block we have
+        should = False
     else:
-        if blacklist.inBlacklist(block_hash): # Dont download blacklisted block
+        if blacklist.inBlacklist(block_hash):
-            ret_data = False
+            # Don't download blacklisted block
-    if ret_data is False:
+            should = False
-        # Remove block from communicator queue if it shouldnt be downloaded
+    if should is False:
+        # Remove block from communicator queue if it shouldn't be downloaded
         try:
             del comm_inst.blockQueue[block_hash]
         except KeyError:
             pass
-    return ret_data
+    return should

src/communicatorutils/uploadblocks/__init__.py

@@ -1,10 +1,22 @@
-'''
+"""Onionr - Private P2P Communication.
-    Onionr - Private P2P Communication
 
-    Upload blocks in the upload queue to peers from the communicator
+Upload blocks in the upload queue to peers from the communicator
-'''
+"""
-from __future__ import annotations
+from typing import TYPE_CHECKING
-'''
+from . import sessionmanager
+
+from onionrtypes import UserID
+import logger
+from communicatorutils import proxypicker
+import onionrexceptions
+from onionrblocks import onionrblockapi as block
+from onionrutils import stringvalidators, basicrequests
+import onionrcrypto
+from communicator import onlinepeers
+if TYPE_CHECKING:
+    from communicator import OnionrCommunicatorDaemon
+
+"""
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation, either version 3 of the License, or
@@ -17,28 +29,23 @@ from __future__ import annotations
 
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
+"""
-from typing import Union, TYPE_CHECKING
-import logger
-from communicatorutils import proxypicker
-import onionrexceptions
-from onionrblocks import onionrblockapi as block
-from onionrutils import localcommand, stringvalidators, basicrequests
-from communicator import onlinepeers
-import onionrcrypto
-from . import sessionmanager
-from . import mixmate
 
-def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
+
-    """Accepts a communicator instance and uploads blocks from its upload queue"""
+def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'):
+    """Accept a communicator instance + upload blocks from its upload queue."""
     """when inserting a block, we try to upload
      it to a few peers to add some deniability & increase functionality"""
     TIMER_NAME = "upload_blocks_from_communicator"
 
-    session_manager: sessionmanager.BlockUploadSessionManager = comm_inst.shared_state.get(sessionmanager.BlockUploadSessionManager)
+    session_manager: sessionmanager.BlockUploadSessionManager
-    triedPeers = []
+    session_manager = comm_inst.shared_state.get(
+        sessionmanager.BlockUploadSessionManager)
+    tried_peers: UserID = []
     finishedUploads = []
-    comm_inst.blocksToUpload = onionrcrypto.cryptoutils.random_shuffle(comm_inst.blocksToUpload)
+    comm_inst.blocksToUpload = onionrcrypto.cryptoutils.random_shuffle(
+        comm_inst.blocksToUpload)
+
     if len(comm_inst.blocksToUpload) != 0:
         for bl in comm_inst.blocksToUpload:
             if not stringvalidators.validate_hash(bl):
@@ -57,21 +64,26 @@ def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
                 except KeyError:
                     pass
                 try:
-                    if session.peer_fails[peer] > 3: continue
+                    if session.peer_fails[peer] > 3:
+                        continue
                 except KeyError:
                     pass
-                if peer in triedPeers: continue
+                if peer in tried_peers:
-                triedPeers.append(peer)
+                    continue
+                tried_peers.append(peer)
                 url = f'http://{peer}/upload'
                 try:
                     data = block.Block(bl).getRaw()
                 except onionrexceptions.NoDataAvailable:
                     finishedUploads.append(bl)
                     break
-                proxyType = proxypicker.pick_proxy(peer)
+                proxy_type = proxypicker.pick_proxy(peer)
-                logger.info(f"Uploading block {bl[:8]} to {peer}", terminal=True)
+                logger.info(
-                resp = basicrequests.do_post_request(url, data=data, proxyType=proxyType, content_type='application/octet-stream')
+                    f"Uploading block {bl[:8]} to {peer}", terminal=True)
-                if not resp == False:
+                resp = basicrequests.do_post_request(
+                    url, data=data, proxyType=proxy_type,
+                    content_type='application/octet-stream')
+                if resp is not False:
                     if resp == 'success':
                         session.success()
                         session.peer_exists[peer] = True
@@ -82,7 +94,9 @@ def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
                         session.fail()
                         session.fail_peer(peer)
                         comm_inst.getPeerProfileInstance(peer).addScore(-5)
-                        logger.warn(f'Failed to upload {bl[:8]}, reason: {resp[:15]}', terminal=True)
+                        logger.warn(
+                            f'Failed to upload {bl[:8]}, reason: {resp[:15]}',
+                            terminal=True)
                 else:
                     session.fail()
         session_manager.clean_session()

src/communicatorutils/uploadblocks/mixmate/pool.py

@@ -62,7 +62,8 @@ class UploadPool:
         """Get the hash pool in secure random order."""
         if len(self._pool) != self._pool_size:
             raise PoolNotReady
-        final_pool: List[onionrtypes.BlockHash] = cryptoutils.random_shuffle(list(self._pool))
+        final_pool: List[onionrtypes.BlockHash] = cryptoutils.random_shuffle(
+            list(self._pool))
 
         self._pool.clear()
         self.birthday = onionrutils.epoch.get_epoch()

src/communicatorutils/uploadblocks/session.py

@@ -2,8 +2,9 @@
 
 Virtual upload "sessions" for blocks
 """
-from typing import Union
+from typing import Union, Dict
 
+from onionrtypes import UserID
 from onionrutils import stringvalidators
 from onionrutils import bytesconverter
 from onionrutils import epoch
@@ -40,8 +41,8 @@ class UploadSession:
         self.block_hash = reconstructhash.deconstruct_hash(block_hash)
         self.total_fail_count: int = 0
         self.total_success_count: int = 0
-        self.peer_fails = {}
+        self.peer_fails: Dict[UserID, int] = {}
-        self.peer_exists = {}
+        self.peer_exists: Dict[UserID, bool] = {}
 
     def fail_peer(self, peer):
         try:

src/communicatorutils/uploadblocks/sessionmanager.py

@@ -1,9 +1,17 @@
-"""
+"""Onionr - Private P2P Communication.
-    Onionr - Private P2P Communication
 
-    Manager for upload 'sessions'
+Manager for upload 'sessions'
 """
-from __future__ import annotations
+from typing import List, Union, TYPE_CHECKING
+if TYPE_CHECKING:
+    from session import UploadSession
+
+from onionrutils import bytesconverter
+from onionrutils import localcommand
+from etc import onionrvalues
+from utils import reconstructhash
+
+from . import session
 """
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -18,31 +26,33 @@ from __future__ import annotations
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 """
-from typing import Iterable, Union
 
-from onionrutils import bytesconverter
-from onionrutils import localcommand
-from etc import onionrvalues
-from etc import waitforsetvar
-from utils import reconstructhash
-
-from . import session
 
 class BlockUploadSessionManager:
-    """Holds block UploadSession instances. Optionally accepts iterable of sessions to added on init
+    """Holds block UploadSession instances.
 
-    Arguments: old_session: iterable of old UploadSession objects"""
+    Optionally accepts iterable of sessions to added on init
-    def __init__(self, old_sessions:Iterable=None):
+    Arguments: old_session: iterable of old UploadSession objects
-        #self._too_many: TooMany = None
+    """
+
+    def __init__(self, old_sessions: List = None):
         if old_sessions is None:
             self.sessions = []
         else:
-            self.sessions = old_session
+            self.sessions = old_sessions
-    
+
-    def add_session(self, session_or_block: Union(str, bytes, session.UploadSession))->session.UploadSession:
+    def add_session(self,
-        """Create (or add existing) block upload session from a str/bytes block hex hash, existing UploadSession"""
+                    session_or_block: Union[str,
-        if isinstance(session_or_block, session.UploadSession): 
+                                            bytes,
-            if not session_or_block in self.sessions:
+                                            session.UploadSession
+                                            ]
+                    ) -> session.UploadSession:
+        """Create (or add existing) block upload session.
+
+        from a str/bytes block hex hash, existing UploadSession
+        """
+        if isinstance(session_or_block, session.UploadSession):
+            if session_or_block not in self.sessions:
                 self.sessions.append(session_or_block)
             return session_or_block
         try:
@@ -50,46 +60,63 @@ class BlockUploadSessionManager:
         except KeyError:
             pass
         # convert bytes hash to str
-        if isinstance(session_or_block, bytes): session_or_block = bytesconverter.bytes_to_str(session_or_block)
+        if isinstance(session_or_block, bytes):
+            session_or_block = bytesconverter.bytes_to_str(session_or_block)
         # intentionally not elif
         if isinstance(session_or_block, str):
             new_session = session.UploadSession(session_or_block)
             self.sessions.append(new_session)
             return new_session
+        raise ValueError
 
-    def get_session(self, block_hash: Union(str, bytes))->session.UploadSession:
+    def get_session(self,
-        block_hash = reconstructhash.deconstruct_hash(bytesconverter.bytes_to_str(block_hash))
+                    block_hash: Union[str, bytes]
-        for session in self.sessions:
+                    ) -> session.UploadSession:
-            if session.block_hash == block_hash: return session
+        block_hash = reconstructhash.deconstruct_hash(
+            bytesconverter.bytes_to_str(block_hash))
+        for sess in self.sessions:
+            if sess.block_hash == block_hash:
+                return sess
         raise KeyError
 
-    def clean_session(self, specific_session: Union[str, UploadSession]=None):
+    def clean_session(self,
-        comm_inst: OnionrCommunicatorDaemon = self._too_many.get_by_string("OnionrCommunicatorDaemon")
+                      specific_session: Union[str, 'UploadSession'] = None):
-        sessions_to_delete = []
-        if comm_inst.getUptime() < 120: return
-        onlinePeerCount = len(comm_inst.onlinePeers)
-        
-        # If we have no online peers right now,
-        if onlinePeerCount == 0: return
 
-        for session in self.sessions:
+        comm_inst: 'OnionrCommunicatorDaemon'  # type: ignore
-            # if over 50% of peers that were online for a session have become unavailable, don't kill sessions
+        comm_inst = self._too_many.get_by_string(  # pylint: disable=E1101 type: ignore
-            if session.total_success_count > onlinePeerCount:
+            "OnionrCommunicatorDaemon")
-                if onlinePeerCount / session.total_success_count >= 0.5: return
+        sessions_to_delete = []
+        if comm_inst.getUptime() < 120:
+            return
+        onlinePeerCount = len(comm_inst.onlinePeers)
+
+        # If we have no online peers right now,
+        if onlinePeerCount == 0:
+            return
+
+        for sess in self.sessions:
+            # if over 50% of peers that were online for a session have
+            # become unavailable, don't kill sessions
+            if sess.total_success_count > onlinePeerCount:
+                if onlinePeerCount / sess.total_success_count >= 0.5:
+                    return
             # Clean sessions if they have uploaded to enough online peers
-            if session.total_success_count <= 0: continue
+            if sess.total_success_count <= 0:
-            if (session.total_success_count / onlinePeerCount) >= onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT:
+                continue
-                sessions_to_delete.append(session)
+            if (sess.total_success_count / onlinePeerCount) >= onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT:
-        for session in sessions_to_delete:
+                sessions_to_delete.append(sess)
+        for sess in sessions_to_delete:
             self.sessions.remove(session)
             # TODO cleanup to one round of search
-            # Remove the blocks from the sessions, upload list, and waitforshare list
+            # Remove the blocks from the sessions, upload list,
+            # and waitforshare list
             try:
-                comm_inst.blocksToUpload.remove(reconstructhash.reconstruct_hash(session.block_hash))
+                comm_inst.blocksToUpload.remove(
+                    reconstructhash.reconstruct_hash(sess.block_hash))
             except ValueError:
                 pass
             try:
-                comm_inst.blocksToUpload.remove(session.block_hash)
+                comm_inst.blocksToUpload.remove(sess.block_hash)
             except ValueError:
                 pass
             localcommand.local_command('waitforshare/{session.block_hash}')