added flood fill work, misc bug fixes and removing communicator timers

src/apiservers/private/register_private_blueprints.py

@@ -5,7 +5,7 @@ This file registers blueprints for the private api server
 from threading import Thread
 from gevent import sleep
 
-from httpapi import security, friendsapi, profilesapi, configapi, insertblock
+from httpapi import security, friendsapi, configapi, insertblock
 from httpapi import miscclientapi, onionrsitesapi, apiutils
 from httpapi import directconnections
 from httpapi import themeapi
@@ -33,7 +33,6 @@ def register_private_blueprints(private_api, app):
     app.register_blueprint(security.client.ClientAPISecurity(
         private_api).client_api_security_bp)
     app.register_blueprint(friendsapi.friends)
-    app.register_blueprint(profilesapi.profile_BP)
     app.register_blueprint(configapi.config_BP)
     app.register_blueprint(insertblock.ib)
     app.register_blueprint(miscclientapi.getblocks.client_get_blocks)

src/communicator/__init__.py

@@ -86,12 +86,10 @@ class OnionrCommunicatorDaemon:
         # extends our upload list and saves our list when Onionr exits
         uploadqueue.UploadQueue(self)
 
-        # Timers to periodically lookup new blocks and download them
-        lookup_blocks_timer = OnionrCommunicatorTimers(
-            self,
+        add_onionr_thread(
             lookupblocks.lookup_blocks_from_communicator,
-            config.get('timers.lookupBlocks', 25),
-            my_args=[self], requires_peer=True, max_threads=1)
+            [self.shared_state], 25, 3)
+
 
         """The block download timer is accessed by the block lookup function
         to trigger faster download starts"""
@@ -99,11 +97,7 @@ class OnionrCommunicatorDaemon:
             self, self.getBlocks, config.get('timers.getBlocks', 10),
             requires_peer=True, max_threads=5)
 
-        # Timer to reset the longest offline peer
-        # so contact can be attempted again
-        OnionrCommunicatorTimers(
-            self, onlinepeers.clear_offline_peer, 58, my_args=[self],
-            max_threads=1)
+        add_onionr_thread(onlinepeers.clear_offline_peer, [self.kv], 58)
 
         # Timer to cleanup old blocks
         blockCleanupTimer = OnionrCommunicatorTimers(
@@ -180,7 +174,6 @@ class OnionrCommunicatorDaemon:
         # Adjust initial timer triggers
         cleanupTimer.count = (cleanupTimer.frequency - 60)
         blockCleanupTimer.count = (blockCleanupTimer.frequency - 2)
-        lookup_blocks_timer = (lookup_blocks_timer.frequency - 2)
 
         shared_state.add(self)
 

src/communicator/onlinepeers/clearofflinepeer.py

@@ -23,9 +23,8 @@ if TYPE_CHECKING:
 """
 
 
-def clear_offline_peer(comm_inst: 'OnionrCommunicatorDaemon'):
+def clear_offline_peer(kv: 'DeadSimpleKV'):
     """Remove the longest offline peer to retry later."""
-    kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string("DeadSimpleKV")
     try:
         removed = kv.get('offlinePeers').pop(0)
     except IndexError:
@@ -33,4 +32,4 @@ def clear_offline_peer(comm_inst: 'OnionrCommunicatorDaemon'):
     else:
         logger.debug('Removed ' + removed +
                      ' from offline list, will try them again.')
-    comm_inst.decrementThreadCount('clear_offline_peer')
+

src/communicatorutils/lookupblocks.py

@@ -39,7 +39,7 @@ blacklist = onionrblacklist.OnionrBlackList()
 storage_counter = StorageCounter()
 
 
-def lookup_blocks_from_communicator(comm_inst):
+def lookup_blocks_from_communicator(shared_state: 'TooMany'):
     logger.info('Looking up new blocks')
     tryAmount = 2
     newBlocks = ''
@@ -50,7 +50,7 @@ def lookup_blocks_from_communicator(comm_inst):
     maxBacklog = 1560
     lastLookupTime = 0  # Last time we looked up a particular peer's list
     new_block_count = 0
-    kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string("DeadSimpleKV")
+    kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
     for i in range(tryAmount):
         # Defined here to reset it each time, time offset is added later
         listLookupCommand = 'getblocklist'
@@ -87,7 +87,7 @@ def lookup_blocks_from_communicator(comm_inst):
         listLookupCommand += '?date=%s' % (lastLookupTime,)
         try:
             newBlocks = peeraction.peer_action(
-                comm_inst.shared_state,
+                shared_state,
                 peer, listLookupCommand)  # get list of new block hashes
         except Exception as error:
             logger.warn(
@@ -124,6 +124,3 @@ def lookup_blocks_from_communicator(comm_inst):
         logger.info(
             f'Discovered {new_block_count} new block{block_string}',
             terminal=True)
-        comm_inst.download_blocks_timer.count = \
-            int(comm_inst.download_blocks_timer.frequency * 0.99)
-    comm_inst.decrementThreadCount('lookup_blocks_from_communicator')

src/httpapi/miscclientapi/endpoints.py

@@ -8,6 +8,7 @@ import platform
 
 from flask import Response, Blueprint, request, send_from_directory, abort
 from flask import g
+from gevent import sleep
 import unpaddedbase32
 
 from httpapi import apiutils
@@ -138,10 +139,6 @@ class PrivateEndpoints:
             return Response(
                 bytesconverter.bytes_to_str(mnemonickeys.get_base32(words)))
 
-        @private_endpoints_bp.route('/gettorsocks')
-        def get_tor_socks():
-            return Response(str(g.too_many.get(NetController).socksPort))
-
         @private_endpoints_bp.route('/setonboarding', methods=['POST'])
         def set_onboarding():
             return Response(
@@ -151,11 +148,25 @@ class PrivateEndpoints:
         def get_os_system():
             return Response(platform.system().lower())
 
+        @private_endpoints_bp.route('/gettorsocks')
+        def get_tor_socks():
+            while True:
+                try:
+                    return Response(
+                        str(
+                            g.too_many.get_by_string(
+                                'NetController').socksPort))
+                except KeyError:
+                    sleep(0.1)
+
         @private_endpoints_bp.route('/torready')
         def is_tor_ready():
             """If Tor is starting up, the web UI is not ready to be used."""
+            try:
                 return Response(
-                str(g.too_many.get(NetController).readyState).lower())
+                    str(g.too_many.get_by_string('NetController').readyState).lower())
+            except KeyError:
+                return Response("false")
 
         @private_endpoints_bp.route('/gettoraddress')
         def get_tor_address():

src/httpapi/profilesapi/__init__.py

@@ -1,27 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    This file creates http endpoints for user profile pages
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-from flask import Blueprint, Response, request, abort
-from . import profiles
-
-profile_BP = Blueprint('profile_BP', __name__)
-
-@profile_BP.route('/profile/get/<pubkey>', endpoint='profiles')
-def get_profile_page(pubkey):
-    return Response(pubkey)

src/httpapi/profilesapi/profiles.py

@@ -1,2 +0,0 @@
-def get_latest_user_profile(pubkey):
-    return ''

src/lan/client/__init__.py

@@ -10,6 +10,7 @@ from onionrtypes import LANIP
 import logger
 from coredb.blockmetadb import get_block_list
 from onionrblocks.blockimporter import import_block_from_data
+import onionrexceptions
 from ..server import ports
 
 from threading import Thread
@@ -37,9 +38,12 @@ def _lan_work(peer: LANIP):
         blocks = requests.get(url + 'blist/0').text.splitlines()
         for block in blocks:
             if block not in our_blocks:
+                try:
                     import_block_from_data(
                         requests.get(
                             url + f'get/{block}', stream=True).raw.read(6000000))
+                except onionrexceptions.InvalidMetadata:
+                    logger.warn(f"Could not get {block} from lan peer")
 
     for port in ports:
         try:

src/onionrthreads/__init__.py

@@ -1,17 +1,25 @@
 from typing import Callable
 from typing import Iterable
 
+import traceback
 from threading import Thread
 
 from time import sleep
 
+import logger
+
 
 def _onionr_thread(func: Callable, args: Iterable,
                    sleep_secs: int, initial_sleep):
     if initial_sleep:
         sleep(initial_sleep)
     while True:
+        try:
             func(*args)
+        except Exception as _:  # noqa
+            logger.warn(
+                "Onionr thread exception \n" + traceback.format_exc(),
+                terminal=True)
         sleep(sleep_secs)
 
 

src/runtests/clearnettor.py

@@ -44,14 +44,14 @@ def test_clearnet_tor_request(testmanager):
 
     try:
         leak_result: str = do_get_request(
-            'https://onionr.net/404',
+            'https://example.com/notvalidpage',
             port=socks_port, ignoreAPI=True).lower()
     except AttributeError:
         leak_result = ""
     except Exception as e:
         logger.warn(str(e))
     try:
-        if 'not found' in leak_result:
+        if 'example' in leak_result:
             logger.error('Tor was able to request a clearnet site')
             raise ValueError('Tor was able to request a clearnet site')
     except TypeError:

src/sneakernet/__init__.py

@@ -45,7 +45,9 @@ class _Importer(FileSystemEventHandler):
         os.remove(event.src_path)
         try:
             import_block_from_data(block_data)
-        except onionrexceptions.DataExists:
+        except(
+                onionrexceptions.DataExists,
+                onionrexceptions.BlockMetaEntryExists) as _:
             return
         if block_data_location in event.src_path:
             try:

src/streamfill/pool.py

@@ -0,0 +1,10 @@
+from typing import List
+
+from onionrutils.localcommand import local_command
+from .neighbors import identify_neighbors
+
+
+def stream_pool():
+
+    peers = lioc
+

src/streamfill/tunnel/__init__.py

@@ -0,0 +1,38 @@
+from secrets import token_bytes
+from typing import TYPE_CHECKING
+import socket
+
+if TYPE_CHECKING:
+    from stem.control import Controller
+
+from onionrtypes import OnionAddressString
+
+import yam
+
+
+def peer_tunnel(tor_controller: Controller, peer):
+    socks_port = tor_controller.get_conf('SocksPort')
+
+    class Connected:
+        connected = False
+
+    send_buffer = []
+    rec_buffer = []
+    rec_address = None
+
+    yam.client(1, peer, socks_port, send_buffer, rec_buffer, Connected)
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        ip = '127.0.0.1'
+        s.bind((ip, 0))
+        s.listen(1)
+        port = s.getsockname()[1]
+        serv = tor_controller.create_ephemeral_hidden_service(
+                {1337: '127.0.0.1:' + str(port)},
+                key_content='ED25519-V3',
+                await_publication=True,
+            )
+        rec_address = serv.service_id
+        conn, addr = s.accept()
+        yam.server(1, tor_controller, conn, send_buffer, rec_buffer, Connected)
+
+

tests/test_random_bind_ip_setting.py

@@ -1,16 +0,0 @@
-import sys, os
-sys.path.append(".")
-sys.path.append("src/")
-import unittest, uuid
-TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
-print("Test directory:", TEST_DIR)
-os.environ["ONIONR_HOME"] = TEST_DIR
-from onionrsetup import setup_config
-setup_config()
-import config
-
-class TestRandomBindIP(unittest.TestCase):
-    def test_random_bind_ip_default_setting(self):
-        self.assertTrue(config.get('general.random_bind_ip'))
-
-unittest.main()