work on torgossip

src/safedb/securestring/__init__.py

@@ -67,8 +67,7 @@ def protect_string(plaintext: Union[bytes, bytearray, str]) -> bytes:
         logger.warn("Error when protecting string for database", terminal=True)
         for line in res[1].decode('utf-8').split('\n'):
             logger.error(line, terminal=True)
-        raise subprocess.CalledProcessError(
+        raise subprocess.CalledProcessError
-            "Error protecting string")
 
 
 def unprotect_string(ciphertext: Union[bytes, bytearray]) -> bytes:
@@ -88,5 +87,4 @@ def unprotect_string(ciphertext: Union[bytes, bytearray]) -> bytes:
             "Error when decrypting ciphertext from database", terminal=True)
         for line in res[1].decode('utf-8').split('\n'):
             logger.error(line, terminal=True)
-        raise subprocess.CalledProcessError(
+        raise subprocess.CalledProcessError
-            "Error unprotecting string")

static-data/default-plugins/torgossip/bootstrap.txt

@@ -1 +1 @@
-wfslzt6nxjjhoslqhgw7qxxet5pkkqb2nedbc44y2577ub2zl32t7tid
+elqy5cbebhcn3xrhkru4odglxiuy5q6dremjq366d53a3c6rbdco6bqd

static-data/default-plugins/torgossip/client.py

@@ -10,6 +10,7 @@ from os import path
 from time import sleep
 from typing import TYPE_CHECKING
 from random import SystemRandom
+from threading import Thread
 
 import socks as socket
 from stem import SocketClosed
@@ -74,10 +75,51 @@ def client_funcs(shared_state, socket_pool):
 
 
     def client_loop(shared_state, socket_pool):
+
         sleep_t = 60
         block_db = shared_state.get_by_string('SafeDB')
         peer_db = shared_state.get_by_string('TorGossipPeers')
 
+        peer_info = {}
+
+        block_types = ['txt', 'bin', 'png']
+
+        peers = list(socket_pool)
+        SystemRandom().shuffle(peers)
+
+        def download_all_missed():
+            while not socket_pool:
+                sleep(4)
+            peer = list(socket_pool)[0]
+            logger.info("[TorGossip] Downloading missed blocks", terminal=True)
+            for bl_type in block_types:
+                while True:
+                    try:
+                        peer_info[peer]
+                    except KeyError:
+                        peer_info[peer] = {'offsets': {}}
+                    else:
+                        try:
+                            peer_info[peer]['offsets']
+                        except KeyError:
+                            peer_info[peer]['offsets'] = {}
+                    try:
+                        offset = peer_info[peer]['offsets'][bl_type]
+                    except KeyError:
+                        offset = 0
+                    try:
+                        peer_info[peer]['offsets'][bl_type] = \
+                            download_blocks(
+                            block_db, socket_pool[peer], offset, bl_type)
+                    except BrokenPipeError:
+                        del peer_info[peer]['offsets']
+                        del socket_pool[peer]
+                    else:
+                        # Go back to for loop
+                        break
+
+        Thread(target=download_all_missed, daemon=True).start()
+
         while True:
             if not socket_pool:
                 try:
@@ -85,11 +127,8 @@ def client_funcs(shared_state, socket_pool):
                 except SocketClosed:  # Probably shutting down, or tor crashed
                     sleep(1)
                     continue
-            peers = list(socket_pool)
-            SystemRandom().shuffle(peers)
             try:
                 peer = peers[0]
-                print(peer)
             except IndexError:
                 logger.error(
                     "There are no known TorGossip peers." +
@@ -97,10 +136,9 @@ def client_funcs(shared_state, socket_pool):
                     terminal=True)
                 sleep(sleep_t)
                 continue
-            try:
+            peers = list(socket_pool)
-                download_blocks(block_db, socket_pool[peer], 0, 'txt')
+            SystemRandom().shuffle(peers)
-            except BrokenPipeError:
+
-                del socket_pool[peer]
 
     _client_pool(shared_state, socket_pool)
     client_loop(shared_state, socket_pool)

static-data/default-plugins/torgossip/clientfuncs/__init__.py

@@ -16,7 +16,8 @@ if TYPE_CHECKING:
     from socket import socket
 
 
-def download_blocks(safe_db, sock: 'socket', offset: int, block_type: str):
+def download_blocks(
+        safe_db, sock: 'socket', offset: int, block_type: str) -> int:
     sock.sendall(
         str(int(GossipCommands.LIST_BLOCKS_BY_TYPE_OFFSET)).encode('utf-8') +
         str(offset).encode('utf-8') + b',' +
@@ -24,12 +25,14 @@ def download_blocks(safe_db, sock: 'socket', offset: int, block_type: str):
     bl_hashs = sock.recv(600000)
     existing_blocks = list_all_blocks(safe_db)
     existing_blocks_hashes = b''
+    downloaded_total = 0  # Including non-succesful
     for i in existing_blocks:
         for x in i:
             existing_blocks_hashes += int(x).to_bytes(1, 'little')
     print('existing', existing_blocks_hashes)
     hash = None
     for i in range(len(bl_hashs)//64):
+        downloaded_total += 1
         hash = bl_hashs[:(i*64) + 64]
         if hash in existing_blocks_hashes:
             continue
@@ -56,3 +59,5 @@ def download_blocks(safe_db, sock: 'socket', offset: int, block_type: str):
         except ValueError:
             #print('not storing dupe block')
             existing_blocks_hashes += hash
+    return downloaded_total
+