work on torgossip

This commit is contained in:
Kevin Froman 2021-02-20 00:41:46 +00:00
parent db5320124f
commit 60dfa8fb7e
4 changed files with 54 additions and 13 deletions

View File

@ -67,8 +67,7 @@ def protect_string(plaintext: Union[bytes, bytearray, str]) -> bytes:
logger.warn("Error when protecting string for database", terminal=True) logger.warn("Error when protecting string for database", terminal=True)
for line in res[1].decode('utf-8').split('\n'): for line in res[1].decode('utf-8').split('\n'):
logger.error(line, terminal=True) logger.error(line, terminal=True)
raise subprocess.CalledProcessError( raise subprocess.CalledProcessError
"Error protecting string")
def unprotect_string(ciphertext: Union[bytes, bytearray]) -> bytes: def unprotect_string(ciphertext: Union[bytes, bytearray]) -> bytes:
@ -88,5 +87,4 @@ def unprotect_string(ciphertext: Union[bytes, bytearray]) -> bytes:
"Error when decrypting ciphertext from database", terminal=True) "Error when decrypting ciphertext from database", terminal=True)
for line in res[1].decode('utf-8').split('\n'): for line in res[1].decode('utf-8').split('\n'):
logger.error(line, terminal=True) logger.error(line, terminal=True)
raise subprocess.CalledProcessError( raise subprocess.CalledProcessError
"Error unprotecting string")

View File

@ -1 +1 @@
wfslzt6nxjjhoslqhgw7qxxet5pkkqb2nedbc44y2577ub2zl32t7tid elqy5cbebhcn3xrhkru4odglxiuy5q6dremjq366d53a3c6rbdco6bqd

View File

@ -10,6 +10,7 @@ from os import path
from time import sleep from time import sleep
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from random import SystemRandom from random import SystemRandom
from threading import Thread
import socks as socket import socks as socket
from stem import SocketClosed from stem import SocketClosed
@ -74,10 +75,51 @@ def client_funcs(shared_state, socket_pool):
def client_loop(shared_state, socket_pool): def client_loop(shared_state, socket_pool):
sleep_t = 60 sleep_t = 60
block_db = shared_state.get_by_string('SafeDB') block_db = shared_state.get_by_string('SafeDB')
peer_db = shared_state.get_by_string('TorGossipPeers') peer_db = shared_state.get_by_string('TorGossipPeers')
peer_info = {}
block_types = ['txt', 'bin', 'png']
peers = list(socket_pool)
SystemRandom().shuffle(peers)
def download_all_missed():
while not socket_pool:
sleep(4)
peer = list(socket_pool)[0]
logger.info("[TorGossip] Downloading missed blocks", terminal=True)
for bl_type in block_types:
while True:
try:
peer_info[peer]
except KeyError:
peer_info[peer] = {'offsets': {}}
else:
try:
peer_info[peer]['offsets']
except KeyError:
peer_info[peer]['offsets'] = {}
try:
offset = peer_info[peer]['offsets'][bl_type]
except KeyError:
offset = 0
try:
peer_info[peer]['offsets'][bl_type] = \
download_blocks(
block_db, socket_pool[peer], offset, bl_type)
except BrokenPipeError:
del peer_info[peer]['offsets']
del socket_pool[peer]
else:
# Go back to for loop
break
Thread(target=download_all_missed, daemon=True).start()
while True: while True:
if not socket_pool: if not socket_pool:
try: try:
@ -85,11 +127,8 @@ def client_funcs(shared_state, socket_pool):
except SocketClosed: # Probably shutting down, or tor crashed except SocketClosed: # Probably shutting down, or tor crashed
sleep(1) sleep(1)
continue continue
peers = list(socket_pool)
SystemRandom().shuffle(peers)
try: try:
peer = peers[0] peer = peers[0]
print(peer)
except IndexError: except IndexError:
logger.error( logger.error(
"There are no known TorGossip peers." + "There are no known TorGossip peers." +
@ -97,10 +136,9 @@ def client_funcs(shared_state, socket_pool):
terminal=True) terminal=True)
sleep(sleep_t) sleep(sleep_t)
continue continue
try: peers = list(socket_pool)
download_blocks(block_db, socket_pool[peer], 0, 'txt') SystemRandom().shuffle(peers)
except BrokenPipeError:
del socket_pool[peer]
_client_pool(shared_state, socket_pool) _client_pool(shared_state, socket_pool)
client_loop(shared_state, socket_pool) client_loop(shared_state, socket_pool)

View File

@ -16,7 +16,8 @@ if TYPE_CHECKING:
from socket import socket from socket import socket
def download_blocks(safe_db, sock: 'socket', offset: int, block_type: str): def download_blocks(
safe_db, sock: 'socket', offset: int, block_type: str) -> int:
sock.sendall( sock.sendall(
str(int(GossipCommands.LIST_BLOCKS_BY_TYPE_OFFSET)).encode('utf-8') + str(int(GossipCommands.LIST_BLOCKS_BY_TYPE_OFFSET)).encode('utf-8') +
str(offset).encode('utf-8') + b',' + str(offset).encode('utf-8') + b',' +
@ -24,12 +25,14 @@ def download_blocks(safe_db, sock: 'socket', offset: int, block_type: str):
bl_hashs = sock.recv(600000) bl_hashs = sock.recv(600000)
existing_blocks = list_all_blocks(safe_db) existing_blocks = list_all_blocks(safe_db)
existing_blocks_hashes = b'' existing_blocks_hashes = b''
downloaded_total = 0 # Including non-succesful
for i in existing_blocks: for i in existing_blocks:
for x in i: for x in i:
existing_blocks_hashes += int(x).to_bytes(1, 'little') existing_blocks_hashes += int(x).to_bytes(1, 'little')
print('existing', existing_blocks_hashes) print('existing', existing_blocks_hashes)
hash = None hash = None
for i in range(len(bl_hashs)//64): for i in range(len(bl_hashs)//64):
downloaded_total += 1
hash = bl_hashs[:(i*64) + 64] hash = bl_hashs[:(i*64) + 64]
if hash in existing_blocks_hashes: if hash in existing_blocks_hashes:
continue continue
@ -56,3 +59,5 @@ def download_blocks(safe_db, sock: 'socket', offset: int, block_type: str):
except ValueError: except ValueError:
#print('not storing dupe block') #print('not storing dupe block')
existing_blocks_hashes += hash existing_blocks_hashes += hash
return downloaded_total