From 4b36e9d3da15418c9f16acb9ce29c1e3288b3bac Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 21 Feb 2021 02:29:00 +0000 Subject: [PATCH] work on torgossip --- scripts/disable-dev-config.py | 1 + scripts/enable-dev-config.py | 1 + .../default-plugins/torgossip/client.py | 8 +++--- .../torgossip/clientfuncs/fanout.py | 27 ++++++++++++++++--- .../default-plugins/torgossip/commands.py | 2 +- .../default-plugins/torgossip/runtest.py | 8 +++--- 6 files changed, 34 insertions(+), 13 deletions(-) diff --git a/scripts/disable-dev-config.py b/scripts/disable-dev-config.py index ca337a12..734b8694 100755 --- a/scripts/disable-dev-config.py +++ b/scripts/disable-dev-config.py @@ -27,6 +27,7 @@ conf['transports']['sneakernet'] = True conf['statistics']['i_dont_want_privacy'] = False conf['statistics']['server'] = '' conf['ui']['animated_background'] = True +conf['runtests']['skip_slow'] = False json.dump(conf, open('static-data/default_config.json', 'w'), sort_keys=True, indent=4) diff --git a/scripts/enable-dev-config.py b/scripts/enable-dev-config.py index b4be790d..6ca06cfb 100755 --- a/scripts/enable-dev-config.py +++ b/scripts/enable-dev-config.py @@ -31,6 +31,7 @@ if input("Use bootstrap list? y/n").lower() == 'y': conf['general']['use_bootstrap_list'] = True conf['log']['file']['remove_on_exit'] = False conf['ui']['animated_background'] = False +conf['runtests']['skip_slow'] = True if input('Stat reporting? y/n') == 'y': conf['statistics']['i_dont_want_privacy'] = True conf['statistics']['server'] = input('Statistics server') diff --git a/static-data/default-plugins/torgossip/client.py b/static-data/default-plugins/torgossip/client.py index f2e4ddee..2652e152 100644 --- a/static-data/default-plugins/torgossip/client.py +++ b/static-data/default-plugins/torgossip/client.py @@ -48,7 +48,8 @@ def client_funcs(shared_state, socket_pool): controller = torcontroller.get_controller() def _client_pool(shared_state, socket_pool: dict): - peer_db: 'TorGossipPeers' = shared_state.get_by_string('TorGossipPeers') + peer_db: 'TorGossipPeers' = shared_state.get_by_string( + 'TorGossipPeers') socks_port = shared_state.get_by_string('NetController').socksPort peers = peer_db.get_highest_score_peers(20) @@ -73,12 +74,10 @@ def client_funcs(shared_state, socket_pool): except socket.GeneralProxyError: s.close() - def client_loop(shared_state, socket_pool): sleep_t = 60 block_db = shared_state.get_by_string('SafeDB') - peer_db = shared_state.get_by_string('TorGossipPeers') peer_info = {} @@ -128,7 +127,7 @@ def client_funcs(shared_state, socket_pool): sleep(1) continue try: - peer = peers[0] + peers[0] except IndexError: logger.error( "There are no known TorGossip peers." + @@ -139,7 +138,6 @@ def client_funcs(shared_state, socket_pool): peers = list(socket_pool) SystemRandom().shuffle(peers) - _client_pool(shared_state, socket_pool) client_loop(shared_state, socket_pool) diff --git a/static-data/default-plugins/torgossip/clientfuncs/fanout.py b/static-data/default-plugins/torgossip/clientfuncs/fanout.py index 87b102c8..5dcedd99 100644 --- a/static-data/default-plugins/torgossip/clientfuncs/fanout.py +++ b/static-data/default-plugins/torgossip/clientfuncs/fanout.py @@ -6,13 +6,17 @@ import os import sys from typing import TYPE_CHECKING from random import SystemRandom +from time import sleep if TYPE_CHECKING: from socket import socket +import logger + sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) from commands import GossipCommands +from .commandsender import command_sender """ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -34,11 +38,26 @@ def fanout_to_peers( block_hash: bytes, block_data: 'KastenPacked', fanout_count: int = 4): - peers_to_use = [] + peers = list(socket_pool) SystemRandom().shuffle(peers) + sent_counter = 0 - for i in range(fanout_count): - peer: 'socket' = peers.pop() - peer.sendall(G) + fanout_count = min(fanout_count, len(peers)) + fanout_count = max(1, fanout_count) + + while sent_counter < fanout_count: + try: + peer = peers.pop() + except IndexError: + sleep(2) + + command_sender(peer, GossipCommands.CHECK_HAS_BLOCK, block_hash) + if peer.recv(1) == b'\x01': + continue + command_sender(peer, GossipCommands.PUT_BLOCK, block_hash, block_data) + if peer.recv(1) != b'\x01': + logger.warn(f"Failed to fanout {block_hash} to {peer}", terminal=True) + continue + sent_counter += 1 diff --git a/static-data/default-plugins/torgossip/commands.py b/static-data/default-plugins/torgossip/commands.py index 1d111ca1..6d4f45f2 100644 --- a/static-data/default-plugins/torgossip/commands.py +++ b/static-data/default-plugins/torgossip/commands.py @@ -23,7 +23,7 @@ along with this program. If not, see . class GossipCommands(IntEnum): PING = 1, - CHECK_HAS_BLOCK = 2, + CHECK_HAS_BLOCK = 2, # Returns 1 if has block, 2 if not LIST_BLOCKS_BY_TYPE = 3, LIST_BLOCKS_BY_TYPE_OFFSET = 4, GET_BLOCK = 5, diff --git a/static-data/default-plugins/torgossip/runtest.py b/static-data/default-plugins/torgossip/runtest.py index 933a7be4..1fe2a74b 100644 --- a/static-data/default-plugins/torgossip/runtest.py +++ b/static-data/default-plugins/torgossip/runtest.py @@ -12,7 +12,7 @@ import blockio def _fake_onion(): - return b32encode(os.urandom(34)).decode('utf-8') + ".onion" + return b32encode(os.urandom(34) + int(3).to_bytes(1, 'little')).decode('utf-8') + ".onion" def _shrink_peer_address(peer): # strip .onion and b32decode peer address for lower database mem usage @@ -70,6 +70,9 @@ def torgossip_runtest(test_manager): # test block was uploaded by getting it s.sendall(b'5' + bl_new.id) assert s.recv(64) == bl_new.get_packed() + # Test CHECK_HASH_BLOCK + s.sendall(b'2' + bl_new.id) + assert s.recv(32) == b'\x02' s.sendall(b'40,tbt') assert len(s.recv(100000)) == len( @@ -104,8 +107,7 @@ def torgossip_runtest(test_manager): announce_raw = _shrink_peer_address(announce_peer) s.sendall(b'8' + announce_raw) assert s.recv(1) == b'1' - print(type(announce_raw), type(shared_state.get_by_string( - 'TorGossipPeers').get_highest_score_peers(100)[0][0])) + assert announce_raw == shared_state.get_by_string( 'TorGossipPeers').get_highest_score_peers(100)[0][0] shared_state.get_by_string('TorGossipPeers').remove_peer(announce_raw)