From bbd76da33301602680997f29d67658b25edb6a7b Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 13 Nov 2020 08:17:48 +0000 Subject: [PATCH] added flood fill work, misc bug fixes and removing communicator timers --- .../private/register_private_blueprints.py | 3 +- src/communicator/__init__.py | 15 ++------ .../onlinepeers/clearofflinepeer.py | 5 +-- src/communicatorutils/lookupblocks.py | 9 ++--- src/httpapi/miscclientapi/endpoints.py | 23 ++++++++--- src/httpapi/profilesapi/__init__.py | 27 ------------- src/httpapi/profilesapi/profiles.py | 2 - src/lan/client/__init__.py | 10 +++-- src/netcontroller/torcontrol/__init__.py | 4 +- src/onionrthreads/__init__.py | 10 ++++- src/runtests/clearnettor.py | 4 +- src/sneakernet/__init__.py | 4 +- src/streamfill/pool.py | 10 +++++ src/streamfill/tunnel/__init__.py | 38 +++++++++++++++++++ tests/test_random_bind_ip_setting.py | 16 -------- 15 files changed, 98 insertions(+), 82 deletions(-) delete mode 100755 src/httpapi/profilesapi/__init__.py delete mode 100755 src/httpapi/profilesapi/profiles.py create mode 100644 src/streamfill/pool.py create mode 100644 src/streamfill/tunnel/__init__.py delete mode 100644 tests/test_random_bind_ip_setting.py diff --git a/src/apiservers/private/register_private_blueprints.py b/src/apiservers/private/register_private_blueprints.py index 28d222ff..4644f87d 100644 --- a/src/apiservers/private/register_private_blueprints.py +++ b/src/apiservers/private/register_private_blueprints.py @@ -5,7 +5,7 @@ This file registers blueprints for the private api server from threading import Thread from gevent import sleep -from httpapi import security, friendsapi, profilesapi, configapi, insertblock +from httpapi import security, friendsapi, configapi, insertblock from httpapi import miscclientapi, onionrsitesapi, apiutils from httpapi import directconnections from httpapi import themeapi @@ -33,7 +33,6 @@ def register_private_blueprints(private_api, app): app.register_blueprint(security.client.ClientAPISecurity( private_api).client_api_security_bp) app.register_blueprint(friendsapi.friends) - app.register_blueprint(profilesapi.profile_BP) app.register_blueprint(configapi.config_BP) app.register_blueprint(insertblock.ib) app.register_blueprint(miscclientapi.getblocks.client_get_blocks) diff --git a/src/communicator/__init__.py b/src/communicator/__init__.py index 77af2bc7..80f8bc6d 100755 --- a/src/communicator/__init__.py +++ b/src/communicator/__init__.py @@ -86,12 +86,10 @@ class OnionrCommunicatorDaemon: # extends our upload list and saves our list when Onionr exits uploadqueue.UploadQueue(self) - # Timers to periodically lookup new blocks and download them - lookup_blocks_timer = OnionrCommunicatorTimers( - self, + add_onionr_thread( lookupblocks.lookup_blocks_from_communicator, - config.get('timers.lookupBlocks', 25), - my_args=[self], requires_peer=True, max_threads=1) + [self.shared_state], 25, 3) + """The block download timer is accessed by the block lookup function to trigger faster download starts""" @@ -99,11 +97,7 @@ class OnionrCommunicatorDaemon: self, self.getBlocks, config.get('timers.getBlocks', 10), requires_peer=True, max_threads=5) - # Timer to reset the longest offline peer - # so contact can be attempted again - OnionrCommunicatorTimers( - self, onlinepeers.clear_offline_peer, 58, my_args=[self], - max_threads=1) + add_onionr_thread(onlinepeers.clear_offline_peer, [self.kv], 58) # Timer to cleanup old blocks blockCleanupTimer = OnionrCommunicatorTimers( @@ -180,7 +174,6 @@ class OnionrCommunicatorDaemon: # Adjust initial timer triggers cleanupTimer.count = (cleanupTimer.frequency - 60) blockCleanupTimer.count = (blockCleanupTimer.frequency - 2) - lookup_blocks_timer = (lookup_blocks_timer.frequency - 2) shared_state.add(self) diff --git a/src/communicator/onlinepeers/clearofflinepeer.py b/src/communicator/onlinepeers/clearofflinepeer.py index d5b1bbd2..47128a73 100644 --- a/src/communicator/onlinepeers/clearofflinepeer.py +++ b/src/communicator/onlinepeers/clearofflinepeer.py @@ -23,9 +23,8 @@ if TYPE_CHECKING: """ -def clear_offline_peer(comm_inst: 'OnionrCommunicatorDaemon'): +def clear_offline_peer(kv: 'DeadSimpleKV'): """Remove the longest offline peer to retry later.""" - kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string("DeadSimpleKV") try: removed = kv.get('offlinePeers').pop(0) except IndexError: @@ -33,4 +32,4 @@ def clear_offline_peer(comm_inst: 'OnionrCommunicatorDaemon'): else: logger.debug('Removed ' + removed + ' from offline list, will try them again.') - comm_inst.decrementThreadCount('clear_offline_peer') + diff --git a/src/communicatorutils/lookupblocks.py b/src/communicatorutils/lookupblocks.py index f0f7e22c..99574b89 100755 --- a/src/communicatorutils/lookupblocks.py +++ b/src/communicatorutils/lookupblocks.py @@ -39,7 +39,7 @@ blacklist = onionrblacklist.OnionrBlackList() storage_counter = StorageCounter() -def lookup_blocks_from_communicator(comm_inst): +def lookup_blocks_from_communicator(shared_state: 'TooMany'): logger.info('Looking up new blocks') tryAmount = 2 newBlocks = '' @@ -50,7 +50,7 @@ def lookup_blocks_from_communicator(comm_inst): maxBacklog = 1560 lastLookupTime = 0 # Last time we looked up a particular peer's list new_block_count = 0 - kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string("DeadSimpleKV") + kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") for i in range(tryAmount): # Defined here to reset it each time, time offset is added later listLookupCommand = 'getblocklist' @@ -87,7 +87,7 @@ def lookup_blocks_from_communicator(comm_inst): listLookupCommand += '?date=%s' % (lastLookupTime,) try: newBlocks = peeraction.peer_action( - comm_inst.shared_state, + shared_state, peer, listLookupCommand) # get list of new block hashes except Exception as error: logger.warn( @@ -124,6 +124,3 @@ def lookup_blocks_from_communicator(comm_inst): logger.info( f'Discovered {new_block_count} new block{block_string}', terminal=True) - comm_inst.download_blocks_timer.count = \ - int(comm_inst.download_blocks_timer.frequency * 0.99) - comm_inst.decrementThreadCount('lookup_blocks_from_communicator') diff --git a/src/httpapi/miscclientapi/endpoints.py b/src/httpapi/miscclientapi/endpoints.py index 09c8e4c5..0eec7d6b 100644 --- a/src/httpapi/miscclientapi/endpoints.py +++ b/src/httpapi/miscclientapi/endpoints.py @@ -8,6 +8,7 @@ import platform from flask import Response, Blueprint, request, send_from_directory, abort from flask import g +from gevent import sleep import unpaddedbase32 from httpapi import apiutils @@ -138,10 +139,6 @@ class PrivateEndpoints: return Response( bytesconverter.bytes_to_str(mnemonickeys.get_base32(words))) - @private_endpoints_bp.route('/gettorsocks') - def get_tor_socks(): - return Response(str(g.too_many.get(NetController).socksPort)) - @private_endpoints_bp.route('/setonboarding', methods=['POST']) def set_onboarding(): return Response( @@ -151,11 +148,25 @@ class PrivateEndpoints: def get_os_system(): return Response(platform.system().lower()) + @private_endpoints_bp.route('/gettorsocks') + def get_tor_socks(): + while True: + try: + return Response( + str( + g.too_many.get_by_string( + 'NetController').socksPort)) + except KeyError: + sleep(0.1) + @private_endpoints_bp.route('/torready') def is_tor_ready(): """If Tor is starting up, the web UI is not ready to be used.""" - return Response( - str(g.too_many.get(NetController).readyState).lower()) + try: + return Response( + str(g.too_many.get_by_string('NetController').readyState).lower()) + except KeyError: + return Response("false") @private_endpoints_bp.route('/gettoraddress') def get_tor_address(): diff --git a/src/httpapi/profilesapi/__init__.py b/src/httpapi/profilesapi/__init__.py deleted file mode 100755 index 1536727b..00000000 --- a/src/httpapi/profilesapi/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -''' - Onionr - Private P2P Communication - - This file creates http endpoints for user profile pages -''' -''' - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -''' -from flask import Blueprint, Response, request, abort -from . import profiles - -profile_BP = Blueprint('profile_BP', __name__) - -@profile_BP.route('/profile/get/', endpoint='profiles') -def get_profile_page(pubkey): - return Response(pubkey) \ No newline at end of file diff --git a/src/httpapi/profilesapi/profiles.py b/src/httpapi/profilesapi/profiles.py deleted file mode 100755 index 11a73b13..00000000 --- a/src/httpapi/profilesapi/profiles.py +++ /dev/null @@ -1,2 +0,0 @@ -def get_latest_user_profile(pubkey): - return '' \ No newline at end of file diff --git a/src/lan/client/__init__.py b/src/lan/client/__init__.py index de57c2dd..5c4f108e 100644 --- a/src/lan/client/__init__.py +++ b/src/lan/client/__init__.py @@ -10,6 +10,7 @@ from onionrtypes import LANIP import logger from coredb.blockmetadb import get_block_list from onionrblocks.blockimporter import import_block_from_data +import onionrexceptions from ..server import ports from threading import Thread @@ -37,9 +38,12 @@ def _lan_work(peer: LANIP): blocks = requests.get(url + 'blist/0').text.splitlines() for block in blocks: if block not in our_blocks: - import_block_from_data( - requests.get( - url + f'get/{block}', stream=True).raw.read(6000000)) + try: + import_block_from_data( + requests.get( + url + f'get/{block}', stream=True).raw.read(6000000)) + except onionrexceptions.InvalidMetadata: + logger.warn(f"Could not get {block} from lan peer") for port in ports: try: diff --git a/src/netcontroller/torcontrol/__init__.py b/src/netcontroller/torcontrol/__init__.py index 90573f38..81f6fb9a 100644 --- a/src/netcontroller/torcontrol/__init__.py +++ b/src/netcontroller/torcontrol/__init__.py @@ -145,8 +145,8 @@ class NetController: except ValueError: # Happens if int() check is not valid logger.error("torPid.txt contained invalid integer. " + - "This indicates corruption " + - "and should not be bypassed for security reasons") + "This indicates corruption " + + "and should not be bypassed for security reasons") return os.remove(self.dataDir + 'torPid.txt') except ProcessLookupError: diff --git a/src/onionrthreads/__init__.py b/src/onionrthreads/__init__.py index 2a0c25c9..48a2c8fd 100644 --- a/src/onionrthreads/__init__.py +++ b/src/onionrthreads/__init__.py @@ -1,17 +1,25 @@ from typing import Callable from typing import Iterable +import traceback from threading import Thread from time import sleep +import logger + def _onionr_thread(func: Callable, args: Iterable, sleep_secs: int, initial_sleep): if initial_sleep: sleep(initial_sleep) while True: - func(*args) + try: + func(*args) + except Exception as _: # noqa + logger.warn( + "Onionr thread exception \n" + traceback.format_exc(), + terminal=True) sleep(sleep_secs) diff --git a/src/runtests/clearnettor.py b/src/runtests/clearnettor.py index 3c7d7119..e17bc9bb 100644 --- a/src/runtests/clearnettor.py +++ b/src/runtests/clearnettor.py @@ -44,14 +44,14 @@ def test_clearnet_tor_request(testmanager): try: leak_result: str = do_get_request( - 'https://onionr.net/404', + 'https://example.com/notvalidpage', port=socks_port, ignoreAPI=True).lower() except AttributeError: leak_result = "" except Exception as e: logger.warn(str(e)) try: - if 'not found' in leak_result: + if 'example' in leak_result: logger.error('Tor was able to request a clearnet site') raise ValueError('Tor was able to request a clearnet site') except TypeError: diff --git a/src/sneakernet/__init__.py b/src/sneakernet/__init__.py index 5d9be61d..0d6f8a98 100644 --- a/src/sneakernet/__init__.py +++ b/src/sneakernet/__init__.py @@ -45,7 +45,9 @@ class _Importer(FileSystemEventHandler): os.remove(event.src_path) try: import_block_from_data(block_data) - except onionrexceptions.DataExists: + except( + onionrexceptions.DataExists, + onionrexceptions.BlockMetaEntryExists) as _: return if block_data_location in event.src_path: try: diff --git a/src/streamfill/pool.py b/src/streamfill/pool.py new file mode 100644 index 00000000..02e56921 --- /dev/null +++ b/src/streamfill/pool.py @@ -0,0 +1,10 @@ +from typing import List + +from onionrutils.localcommand import local_command +from .neighbors import identify_neighbors + + +def stream_pool(): + + peers = lioc + diff --git a/src/streamfill/tunnel/__init__.py b/src/streamfill/tunnel/__init__.py new file mode 100644 index 00000000..762976e2 --- /dev/null +++ b/src/streamfill/tunnel/__init__.py @@ -0,0 +1,38 @@ +from secrets import token_bytes +from typing import TYPE_CHECKING +import socket + +if TYPE_CHECKING: + from stem.control import Controller + +from onionrtypes import OnionAddressString + +import yam + + +def peer_tunnel(tor_controller: Controller, peer): + socks_port = tor_controller.get_conf('SocksPort') + + class Connected: + connected = False + + send_buffer = [] + rec_buffer = [] + rec_address = None + + yam.client(1, peer, socks_port, send_buffer, rec_buffer, Connected) + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + ip = '127.0.0.1' + s.bind((ip, 0)) + s.listen(1) + port = s.getsockname()[1] + serv = tor_controller.create_ephemeral_hidden_service( + {1337: '127.0.0.1:' + str(port)}, + key_content='ED25519-V3', + await_publication=True, + ) + rec_address = serv.service_id + conn, addr = s.accept() + yam.server(1, tor_controller, conn, send_buffer, rec_buffer, Connected) + + diff --git a/tests/test_random_bind_ip_setting.py b/tests/test_random_bind_ip_setting.py deleted file mode 100644 index 59bee035..00000000 --- a/tests/test_random_bind_ip_setting.py +++ /dev/null @@ -1,16 +0,0 @@ -import sys, os -sys.path.append(".") -sys.path.append("src/") -import unittest, uuid -TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/' -print("Test directory:", TEST_DIR) -os.environ["ONIONR_HOME"] = TEST_DIR -from onionrsetup import setup_config -setup_config() -import config - -class TestRandomBindIP(unittest.TestCase): - def test_random_bind_ip_default_setting(self): - self.assertTrue(config.get('general.random_bind_ip')) - -unittest.main()