From 60dfa8fb7efce3ba24811d39981ce617958a8528 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 20 Feb 2021 00:41:46 +0000 Subject: [PATCH] work on torgossip --- src/safedb/securestring/__init__.py | 6 +-- .../default-plugins/torgossip/bootstrap.txt | 2 +- .../default-plugins/torgossip/client.py | 52 ++++++++++++++++--- .../torgossip/clientfuncs/__init__.py | 7 ++- 4 files changed, 54 insertions(+), 13 deletions(-) diff --git a/src/safedb/securestring/__init__.py b/src/safedb/securestring/__init__.py index 01731dde..b9acf637 100644 --- a/src/safedb/securestring/__init__.py +++ b/src/safedb/securestring/__init__.py @@ -67,8 +67,7 @@ def protect_string(plaintext: Union[bytes, bytearray, str]) -> bytes: logger.warn("Error when protecting string for database", terminal=True) for line in res[1].decode('utf-8').split('\n'): logger.error(line, terminal=True) - raise subprocess.CalledProcessError( - "Error protecting string") + raise subprocess.CalledProcessError def unprotect_string(ciphertext: Union[bytes, bytearray]) -> bytes: @@ -88,5 +87,4 @@ def unprotect_string(ciphertext: Union[bytes, bytearray]) -> bytes: "Error when decrypting ciphertext from database", terminal=True) for line in res[1].decode('utf-8').split('\n'): logger.error(line, terminal=True) - raise subprocess.CalledProcessError( - "Error unprotecting string") + raise subprocess.CalledProcessError diff --git a/static-data/default-plugins/torgossip/bootstrap.txt b/static-data/default-plugins/torgossip/bootstrap.txt index d47a5dab..c94cd6e7 100644 --- a/static-data/default-plugins/torgossip/bootstrap.txt +++ b/static-data/default-plugins/torgossip/bootstrap.txt @@ -1 +1 @@ -wfslzt6nxjjhoslqhgw7qxxet5pkkqb2nedbc44y2577ub2zl32t7tid \ No newline at end of file +elqy5cbebhcn3xrhkru4odglxiuy5q6dremjq366d53a3c6rbdco6bqd \ No newline at end of file diff --git a/static-data/default-plugins/torgossip/client.py b/static-data/default-plugins/torgossip/client.py index c576c1a6..f2e4ddee 100644 --- a/static-data/default-plugins/torgossip/client.py +++ b/static-data/default-plugins/torgossip/client.py @@ -10,6 +10,7 @@ from os import path from time import sleep from typing import TYPE_CHECKING from random import SystemRandom +from threading import Thread import socks as socket from stem import SocketClosed @@ -74,10 +75,51 @@ def client_funcs(shared_state, socket_pool): def client_loop(shared_state, socket_pool): + sleep_t = 60 block_db = shared_state.get_by_string('SafeDB') peer_db = shared_state.get_by_string('TorGossipPeers') + peer_info = {} + + block_types = ['txt', 'bin', 'png'] + + peers = list(socket_pool) + SystemRandom().shuffle(peers) + + def download_all_missed(): + while not socket_pool: + sleep(4) + peer = list(socket_pool)[0] + logger.info("[TorGossip] Downloading missed blocks", terminal=True) + for bl_type in block_types: + while True: + try: + peer_info[peer] + except KeyError: + peer_info[peer] = {'offsets': {}} + else: + try: + peer_info[peer]['offsets'] + except KeyError: + peer_info[peer]['offsets'] = {} + try: + offset = peer_info[peer]['offsets'][bl_type] + except KeyError: + offset = 0 + try: + peer_info[peer]['offsets'][bl_type] = \ + download_blocks( + block_db, socket_pool[peer], offset, bl_type) + except BrokenPipeError: + del peer_info[peer]['offsets'] + del socket_pool[peer] + else: + # Go back to for loop + break + + Thread(target=download_all_missed, daemon=True).start() + while True: if not socket_pool: try: @@ -85,11 +127,8 @@ def client_funcs(shared_state, socket_pool): except SocketClosed: # Probably shutting down, or tor crashed sleep(1) continue - peers = list(socket_pool) - SystemRandom().shuffle(peers) try: peer = peers[0] - print(peer) except IndexError: logger.error( "There are no known TorGossip peers." + @@ -97,10 +136,9 @@ def client_funcs(shared_state, socket_pool): terminal=True) sleep(sleep_t) continue - try: - download_blocks(block_db, socket_pool[peer], 0, 'txt') - except BrokenPipeError: - del socket_pool[peer] + peers = list(socket_pool) + SystemRandom().shuffle(peers) + _client_pool(shared_state, socket_pool) client_loop(shared_state, socket_pool) diff --git a/static-data/default-plugins/torgossip/clientfuncs/__init__.py b/static-data/default-plugins/torgossip/clientfuncs/__init__.py index aa252c02..5c92ecd6 100644 --- a/static-data/default-plugins/torgossip/clientfuncs/__init__.py +++ b/static-data/default-plugins/torgossip/clientfuncs/__init__.py @@ -16,7 +16,8 @@ if TYPE_CHECKING: from socket import socket -def download_blocks(safe_db, sock: 'socket', offset: int, block_type: str): +def download_blocks( + safe_db, sock: 'socket', offset: int, block_type: str) -> int: sock.sendall( str(int(GossipCommands.LIST_BLOCKS_BY_TYPE_OFFSET)).encode('utf-8') + str(offset).encode('utf-8') + b',' + @@ -24,12 +25,14 @@ def download_blocks(safe_db, sock: 'socket', offset: int, block_type: str): bl_hashs = sock.recv(600000) existing_blocks = list_all_blocks(safe_db) existing_blocks_hashes = b'' + downloaded_total = 0 # Including non-succesful for i in existing_blocks: for x in i: existing_blocks_hashes += int(x).to_bytes(1, 'little') print('existing', existing_blocks_hashes) hash = None for i in range(len(bl_hashs)//64): + downloaded_total += 1 hash = bl_hashs[:(i*64) + 64] if hash in existing_blocks_hashes: continue @@ -56,3 +59,5 @@ def download_blocks(safe_db, sock: 'socket', offset: int, block_type: str): except ValueError: #print('not storing dupe block') existing_blocks_hashes += hash + return downloaded_total +