From db5320124fe14ef5b53affab5e140a24975db386 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 16 Feb 2021 05:01:01 +0000 Subject: [PATCH] work on torgossip --- .../daemonlaunch/spawndaemonthreads.py | 3 ++ .../default-plugins/torgossip/client.py | 2 +- .../torgossip/clientfuncs/__init__.py | 42 ++++++++++++++++--- static-data/default-plugins/torgossip/main.py | 1 - .../default-plugins/torgossip/server.py | 2 +- 5 files changed, 41 insertions(+), 9 deletions(-) diff --git a/src/onionrcommands/daemonlaunch/spawndaemonthreads.py b/src/onionrcommands/daemonlaunch/spawndaemonthreads.py index 86d4efac..48c48b59 100644 --- a/src/onionrcommands/daemonlaunch/spawndaemonthreads.py +++ b/src/onionrcommands/daemonlaunch/spawndaemonthreads.py @@ -12,6 +12,7 @@ from communicatorutils import uploadblocks from communicatorutils import announcenode, deniableinserts from communicatorutils import netcheck import onionrpeers +from blockio import clean_expired_blocks import config @@ -69,3 +70,5 @@ def spawn_client_threads(shared_state: 'TooMany'): add_onionr_thread( announcenode.announce_node, [shared_state], 600, 60) add_onionr_thread(onionrpeers.peer_cleanup, [], 300, 300) + + add_onionr_thread(clean_expired_blocks, [shared_state.get_by_string('SafeDB')], 120, 1) diff --git a/static-data/default-plugins/torgossip/client.py b/static-data/default-plugins/torgossip/client.py index ee49d130..c576c1a6 100644 --- a/static-data/default-plugins/torgossip/client.py +++ b/static-data/default-plugins/torgossip/client.py @@ -98,7 +98,7 @@ def client_funcs(shared_state, socket_pool): sleep(sleep_t) continue try: - download_blocks(socket_pool[peer], 0, 'txt') + download_blocks(block_db, socket_pool[peer], 0, 'txt') except BrokenPipeError: del socket_pool[peer] diff --git a/static-data/default-plugins/torgossip/clientfuncs/__init__.py b/static-data/default-plugins/torgossip/clientfuncs/__init__.py index de545a8f..aa252c02 100644 --- a/static-data/default-plugins/torgossip/clientfuncs/__init__.py +++ b/static-data/default-plugins/torgossip/clientfuncs/__init__.py @@ -2,9 +2,13 @@ from typing import TYPE_CHECKING import sys import os +from kasten import Kasten +from kasten.generator import pack + import logger -from blockio import store_block, subprocvalidate +from blockio import store_block, subprocvalidate, list_all_blocks import onionrblocks +from onionrblocks.exceptions import BlockExpired sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) from commands import GossipCommands @@ -12,17 +16,43 @@ if TYPE_CHECKING: from socket import socket -def download_blocks(sock: 'socket', offset: int, block_type: str): +def download_blocks(safe_db, sock: 'socket', offset: int, block_type: str): sock.sendall( - str(int(GossipCommands.LIST_BLOCKS_BY_TYPE_OFFSET)).encode('utf-8') + str(offset).encode('utf-8') + b',' + - block_type.encode('utf-8')) + str(int(GossipCommands.LIST_BLOCKS_BY_TYPE_OFFSET)).encode('utf-8') + + str(offset).encode('utf-8') + b',' + + block_type.encode('utf-8')) bl_hashs = sock.recv(600000) + existing_blocks = list_all_blocks(safe_db) + existing_blocks_hashes = b'' + for i in existing_blocks: + for x in i: + existing_blocks_hashes += int(x).to_bytes(1, 'little') + print('existing', existing_blocks_hashes) hash = None for i in range(len(bl_hashs)//64): hash = bl_hashs[:(i*64) + 64] + if hash in existing_blocks_hashes: + continue sock.sendall( str(int(GossipCommands.GET_BLOCK)).encode('utf-8') + hash) bl_content = sock.recv(10**6) + if bl_content == b'0': + existing_blocks.append(hash) + continue print('got block', bl_content) - - + try: + store_block( + Kasten( + hash, + bl_content, + generator=onionrblocks.generators.AnonVDFGenerator), + safe_db + ) + existing_blocks_hashes += hash + print('stored block!') + except BlockExpired: + print('Block expired', hash) + existing_blocks_hashes += hash + except ValueError: + #print('not storing dupe block') + existing_blocks_hashes += hash diff --git a/static-data/default-plugins/torgossip/main.py b/static-data/default-plugins/torgossip/main.py index 3940841c..c076f9f0 100755 --- a/static-data/default-plugins/torgossip/main.py +++ b/static-data/default-plugins/torgossip/main.py @@ -71,7 +71,6 @@ def on_init(api, data=None): hf.write(hs) logger.info("TorGossip server on " + b32encode(hs).lower().decode('utf-8'), terminal=True) - Thread(target=start_server, daemon=True, args=[shared_state]).start() Thread(target=start_client, daemon=True, args=[shared_state]).start() diff --git a/static-data/default-plugins/torgossip/server.py b/static-data/default-plugins/torgossip/server.py index 849551bc..c04ea68d 100644 --- a/static-data/default-plugins/torgossip/server.py +++ b/static-data/default-plugins/torgossip/server.py @@ -38,7 +38,7 @@ along with this program. If not, see . def start_server(shared_state): - bl = subprocgenerate.vdf_block(b"yep", "txt", 120) + bl = subprocgenerate.vdf_block(b"yep" + os.urandom(5), "txt", 6000) store_block(Kasten(bl.id, bl.get_packed(), generator=AnonVDFGenerator), shared_state.get_by_string('SafeDB')) print(bl.id)