dont upload blocks that arent valid (anymore?), progress removing communicator timers

scripts/disable-dev-config.py

@@ -16,6 +16,7 @@ conf['general']['insert_deniable_blocks'] = True
 conf['general']['random_bind_ip'] = True
 conf['general']['display_header'] = True
 conf['general']['security_level'] = 0
+conf['general']['use_bootstrap_list'] = True
 conf['onboarding']['done'] = False
 conf['general']['minimum_block_pow'] = 5
 conf['general']['minimum_send_pow'] = 5

scripts/enable-dev-config.py

@@ -26,6 +26,9 @@ conf['general']['random_bind_ip'] = False
 conf['onboarding']['done'] = True
 conf['general']['minimum_block_pow'] = block_pow
 conf['general']['minimum_send_pow'] = block_pow
+conf['general']['use_bootstrap_list'] = False
+if input("Use bootstrap list? y/n").lower() == 'y':
+    conf['general']['use_bootstrap_list'] = True
 conf['log']['file']['remove_on_exit'] = False
 conf['ui']['animated_background'] = False
 if input('Stat reporting? y/n') == 'y':

src/communicator/__init__.py

@@ -57,6 +57,7 @@ class OnionrCommunicatorDaemon:
         # configure logger and stuff
         self.config = config
         self.shared_state = shared_state  # TooManyObjects module
+        shared_state.add(self)
 
         # populate kv values
         self.kv = self.shared_state.get_by_string('DeadSimpleKV')
@@ -98,7 +99,7 @@ class OnionrCommunicatorDaemon:
         add_onionr_thread(onlinepeers.clear_offline_peer, [self.kv], 58)
 
         add_onionr_thread(
-            housekeeping.clean_old_blocks, [self.shared_state], 20, 1)
+            housekeeping.clean_old_blocks, [self.shared_state], 10, 1)
 
         # Discover new peers
         add_onionr_thread(
@@ -108,12 +109,13 @@ class OnionrCommunicatorDaemon:
         # Timer for adjusting which peers
         # we actively communicate to at any given time,
         # to avoid over-using peers
-        add_onionr_thread(cooldownpeer.cooldown_peer, [self.shared_state], 30, 60)
+        add_onionr_thread(
+            cooldownpeer.cooldown_peer, [self.shared_state], 30, 60)
 
         # Timer to read the upload queue and upload the entries to peers
-        OnionrCommunicatorTimers(
+        add_onionr_thread(
-            self, uploadblocks.upload_blocks_from_communicator,
+            uploadblocks.upload_blocks_from_communicator,
-            5, my_args=[self], requires_peer=True, max_threads=1)
+            [self.shared_state], 5, 1)
 
         # Setup direct connections
         if config.get('general.ephemeral_tunnels', False):
@@ -132,10 +134,7 @@ class OnionrCommunicatorDaemon:
         # This timer creates deniable blocks,
         # in an attempt to further obfuscate block insertion metadata
         if config.get('general.insert_deniable_blocks', True):
-            deniableBlockTimer = OnionrCommunicatorTimers(
+            add_onionr_thread(deniableinserts.insert_deniable_block, [], 180, 10)
-                self, deniableinserts.insert_deniable_block,
-                180, my_args=[self], requires_peer=True, max_threads=1)
-            deniableBlockTimer.count = (deniableBlockTimer.frequency - 175)
 
         if config.get('transports.tor', True):
             # Timer to check for connectivity,
@@ -168,8 +167,6 @@ class OnionrCommunicatorDaemon:
         # Adjust initial timer triggers
         cleanupTimer.count = (cleanupTimer.frequency - 60)
 
-        shared_state.add(self)
-
         if config.get('general.use_bootstrap_list', True):
             bootstrappeers.add_bootstrap_list_to_peer_list(
                 self.kv, [], db_only=True)

src/communicatorutils/deniableinserts.py

@@ -22,7 +22,7 @@ import onionrblocks
 """
 
 
-def insert_deniable_block(comm_inst):
+def insert_deniable_block():
     """Insert a fake block to make it more difficult to track real blocks."""
     fakePeer = ''
     chance = 10
@@ -33,4 +33,3 @@ def insert_deniable_block(comm_inst):
         onionrblocks.insert(data, header='pm', encryptType='asym',
                             asymPeer=fakePeer, disableForward=True,
                             meta={'subject': 'foo'})
-    comm_inst.decrementThreadCount('insert_deniable_block')

src/communicatorutils/uploadblocks/__init__.py

@@ -14,7 +14,9 @@ import logger
 from communicatorutils import proxypicker
 import onionrexceptions
 from onionrblocks import onionrblockapi as block
+from onionrblocks.blockmetadata.fromdata import get_block_metadata_from_data
 from onionrutils import stringvalidators, basicrequests
+from onionrutils.validatemetadata import validate_metadata
 from communicator import onlinepeers
 if TYPE_CHECKING:
     from deadsimplekv import DeadSimpleKV
@@ -35,15 +37,14 @@ if TYPE_CHECKING:
 """
 
 
-def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'):
+def upload_blocks_from_communicator(shared_state: 'OnionrCommunicatorDaemon'):
     """Accept a communicator instance + upload blocks from its upload queue."""
     """when inserting a block, we try to upload
      it to a few peers to add some deniability & increase functionality"""
-    kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string("DeadSimpleKV")
+    kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
-    TIMER_NAME = "upload_blocks_from_communicator"
 
     session_manager: sessionmanager.BlockUploadSessionManager
-    session_manager = comm_inst.shared_state.get(
+    session_manager = shared_state.get(
         sessionmanager.BlockUploadSessionManager)
     tried_peers: UserID = []
     finishedUploads = []
@@ -53,7 +54,7 @@ def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'):
     def remove_from_hidden(bl):
         sleep(60)
         try:
-            comm_inst.shared_state.get_by_string(
+            shared_state.get_by_string(
                 'PublicAPI').hideBlocks.remove(bl)
         except ValueError:
             pass
@@ -62,7 +63,6 @@ def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'):
         for bl in kv.get('blocksToUpload'):
             if not stringvalidators.validate_hash(bl):
                 logger.warn('Requested to upload invalid block', terminal=True)
-                comm_inst.decrementThreadCount(TIMER_NAME)
                 return
             session = session_manager.add_session(bl)
             for _ in range(min(len(kv.get('onlinePeers')), 6)):
@@ -92,10 +92,22 @@ def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'):
                     data = block.Block(bl).getRaw()
                     if not data:
                         logger.warn(
-                            f"Couldn't data for block in upload list {bl}",
+                            f"Couldn't get data for block in upload list {bl}",
                             terminal=True)
                         raise onionrexceptions.NoDataAvailable
-                except onionrexceptions.NoDataAvailable:
+                    try:
+                        def __check_metadata():
+                            metadata = get_block_metadata_from_data(data)[0]
+                            if not validate_metadata(metadata, data):
+                                logger.warn(
+                                    f"Metadata for uploading block not valid {bl}")
+                                raise onionrexceptions.InvalidMetadata
+                        __check_metadata()
+                    except onionrexceptions.DataExists:
+                        pass
+                except(  # noqa
+                        onionrexceptions.NoDataAvailable,
+                        onionrexceptions.InvalidMetadata) as _:
                     finishedUploads.append(bl)
                     break
                 proxy_type = proxypicker.pick_proxy(peer)
@@ -116,7 +128,9 @@ def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'):
                     else:
                         session.fail()
                         session.fail_peer(peer)
-                        comm_inst.getPeerProfileInstance(peer).addScore(-5)
+                        shared_state.get_by_string(
+                            'OnionrCommunicatorDaemon').getPeerProfileInstance(
+                                peer).addScore(-5)
                         logger.warn(
                            f'Failed to upload {bl[:8]}, reason: {resp}',
                            terminal=True)
@@ -127,9 +141,8 @@ def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'):
         try:
             kv.get('blocksToUpload').remove(x)
 
-            comm_inst.shared_state.get_by_string(
+            shared_state.get_by_string(
                 'PublicAPI').hideBlocks.remove(x)
 
         except ValueError:
-            pass
+            pass
-    comm_inst.decrementThreadCount(TIMER_NAME)

src/httpapi/miscclientapi/endpoints.py

@@ -116,6 +116,8 @@ class PrivateEndpoints:
                         SerializedData).get_stats())
                 except AttributeError:
                     pass
+                except FileNotFoundError:
+                    pass
 
         @private_endpoints_bp.route('/getuptime')
         def show_uptime():

src/httpapi/miscpublicapi/upload.py

@@ -71,6 +71,10 @@ def accept_upload(request):
         except onionrexceptions.PlaintextNotSupported:
             logger.debug("attempted plaintext upload to us: {b_hash}")
             resp = 'failure'
+        except onionrexceptions.InvalidMetadata:
+            logger.debug(
+                'uploaded block {b_hash} has invalid metadata')
+            resp = 'failure'
     if resp == 'failure':
         abort(400)
     elif resp == 'size':