From 5ea90acd3f0930947373f8d3adf728b8c759a786 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 2 Feb 2021 23:24:35 +0000 Subject: [PATCH] work on torgossip --- scripts/generate-onions.py | 47 +++++---------- static-data/base64-code-whitelist.txt | 1 + .../default-plugins/torgossip/client.py | 3 +- .../torgossip/commandhandlers.py | 7 +++ .../default-plugins/torgossip/commands.py | 3 +- static-data/default-plugins/torgossip/main.py | 4 ++ .../default-plugins/torgossip/peerdb.py | 42 ++++++++++--- .../default-plugins/torgossip/runtest.py | 19 +++--- .../benchmark-neighbor-closeness.py | 34 ----------- tests/test_streamfill_extract_ed25519.py | 53 ---------------- tests/test_streamfill_neighbors.py | 60 ------------------- 11 files changed, 77 insertions(+), 196 deletions(-) delete mode 100644 tests/benchmarks/benchmark-neighbor-closeness.py delete mode 100644 tests/test_streamfill_extract_ed25519.py delete mode 100644 tests/test_streamfill_neighbors.py diff --git a/scripts/generate-onions.py b/scripts/generate-onions.py index 414e9c89..e8fa03d7 100755 --- a/scripts/generate-onions.py +++ b/scripts/generate-onions.py @@ -2,40 +2,21 @@ import sys import os -import stem -from stem import process -from stem.control import Controller -if not os.path.exists('onionr.sh'): - os.chdir('../') -sys.path.append("src/") +from base64 import b32encode +from hashlib import sha3_256 try: - sys.argv[1] + amount = int(sys.argv[1]) except IndexError: - sys.exit(1) + amount = 1 -tor_process = process.launch_tor_with_config( - completion_percent=0, - config = { - 'ControlPort': '2778', - 'DisableNetwork': '1', - 'Log': [ - 'NOTICE stdout', - 'ERR file /tmp/tor_error_log', - ], - }, -) - -with Controller.from_port('127.0.0.1', 2778) as controller: - controller.authenticate() - for i in range(1024, 1024 + int(sys.argv[1])): - hs = controller.create_ephemeral_hidden_service( - {80: i}, - key_type='NEW', - key_content='ED25519-V3', - await_publication=False, - detached=True) - print(hs.service_id + ".onion") - controller.remove_ephemeral_hidden_service(hs.service_id) - -tor_process.kill() \ No newline at end of file +version = int(3).to_bytes(1, "little") +for i in range(amount): + pubkey = os.urandom(32) + #digest = sha3_256(b".onion checksum" + pubkey + version).digest()[:2] + digest = sha3_256() + digest.update(b".onion checksum") + digest.update(pubkey) + digest.update(version) + digest = digest.digest()[:2] + print(b32encode(pubkey + digest + version).decode().lower() + ".onion") diff --git a/static-data/base64-code-whitelist.txt b/static-data/base64-code-whitelist.txt index 30c0cf63..cfc77904 100644 --- a/static-data/base64-code-whitelist.txt +++ b/static-data/base64-code-whitelist.txt @@ -67,3 +67,4 @@ ZABkAWwAWgBkAGQBbAFaAWQAZAJsAm0DWgMBAGQAZANsBG0FWgUBAGQAZARsBm0HWgcBAGQFZAaEAFoI ZABkAWwAWgBkAGQBbAFaAWQAZAJsAm0DWgMBAGQAZANsBG0FWgUBAGQAZARsBm0HWgcBAGQAZAVsCG0JWgkBAGQAZAZsCm0LWgsBAGQHZAiEAFoMZAFTAA== ZABkAWwAWgBkAGQBbAFaAWQAZAJsAm0DWgMBAGQAZANsBG0FWgUBAGQAZARsBm0HWgcBAGQAZAVsCG0JWgkBAGQAZAZsCm0LWgsBAGQAZAFsCFoIZAdkCIQAWgxkAVMA ZABaAGQBZAJsAW0CWgIBAGQBZANsA20EWgQBAGQBZARsBVoFZAFkBGwBWgFkAWQFbAZtB1oHAQBkBmQHhABaCGUJZAicAWQJZAqEBFoKZQlkCJwBZAtkDIQEWgtkDWQOhABaDGQEUwA= +ZABaAGQBZAJsAW0CWgIBAGQBZANsA20EWgQBAGQBZARsBW0GWgYBAGQBZAVsB1oHZAFkBWwDWgNkAWQGbAhtCVoJAQBkB2QIhABaCmULZAmcAWQKZAuEBFoMZAxkDYQAWg1lC2QJnAFkDmQPhARaDmQQZBGEAFoPZAVTAA== diff --git a/static-data/default-plugins/torgossip/client.py b/static-data/default-plugins/torgossip/client.py index 7a1d10f2..0c7dbd6b 100644 --- a/static-data/default-plugins/torgossip/client.py +++ b/static-data/default-plugins/torgossip/client.py @@ -21,5 +21,6 @@ along with this program. If not, see . def client_pool(shared_state): + return + - diff --git a/static-data/default-plugins/torgossip/commandhandlers.py b/static-data/default-plugins/torgossip/commandhandlers.py index 548428e8..6aa0221f 100644 --- a/static-data/default-plugins/torgossip/commandhandlers.py +++ b/static-data/default-plugins/torgossip/commandhandlers.py @@ -9,6 +9,7 @@ import blockio import onionrblocks from kasten import Kasten + """ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -25,6 +26,12 @@ along with this program. If not, see . """ +def peer_exchange(peers: 'Peers', num_of_peers: bytes): + #7 + num_of_peers = int.from_bytes(num_of_peers, 'little') + return peers.get_highest_score_peers(num_of_peers) + + def put_block(safe_db, block): #6 block_hash = block[:64] diff --git a/static-data/default-plugins/torgossip/commands.py b/static-data/default-plugins/torgossip/commands.py index 2e02150e..a2fe686d 100644 --- a/static-data/default-plugins/torgossip/commands.py +++ b/static-data/default-plugins/torgossip/commands.py @@ -28,4 +28,5 @@ class GossipCommands(IntEnum): GET_BLOCK = 5, PUT_BLOCK = 6, PEER_EXCHANGE = 7, - EXIT = 8 + ANNOUNCE_PEER = 8, + EXIT = 9 diff --git a/static-data/default-plugins/torgossip/main.py b/static-data/default-plugins/torgossip/main.py index dd982de5..f660946c 100755 --- a/static-data/default-plugins/torgossip/main.py +++ b/static-data/default-plugins/torgossip/main.py @@ -24,12 +24,16 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . """ plugin_name = 'torgossip' + from server import start_server +from peerdb import Peers from runtest import torgossip_runtest def on_init(api, data=None): shared_state = data + shared_state.get(Peers) + shared_state.get_by_string( "OnionrRunTestManager").plugin_tests.append(torgossip_runtest) diff --git a/static-data/default-plugins/torgossip/peerdb.py b/static-data/default-plugins/torgossip/peerdb.py index 23bf263b..c4a76eb6 100644 --- a/static-data/default-plugins/torgossip/peerdb.py +++ b/static-data/default-plugins/torgossip/peerdb.py @@ -26,11 +26,11 @@ along with this program. If not, see . -class Peers: +class TorGossipPeers: # name it this way to avoid collisions in SharedState PACK_FORMAT = "qQ" - def __init__(self, encrypt: bool): + def __init__(self, encrypt=True): self.db = safedb.SafeDB( identify_home() + "/torgossip-peers.db", protected=encrypt) @@ -42,8 +42,11 @@ class Peers: def _pack_and_store_info(self, peer, new_score=None, new_seen=None): # Repack peer information with new value(s) and store it - score, seen = unpack(self.PACK_FORMAT, self.db.get( - self._shrink_peer_address(peer))) + peer = self._shrink_peer_address(peer) + try: + score, seen = unpack(self.PACK_FORMAT, self.db.get(peer)) + except KeyError: + score = seen = 0 if new_score: score = new_score @@ -52,6 +55,31 @@ class Peers: self.db.put(peer, pack(self.PACK_FORMAT, score, seen)) + def get_highest_score_peers(self, max=5): + assert max >= 1 + peer = self.db.db_conn.firstkey() + + top = [(peer, self.db.get(peer))] + + for peer in self.db.db_conn.nextkey(peer): + peer_data = self.db.get(peer) + overwrite = None + + if len(top) != max: + top.append((peer, peer_data)) + continue + for count, cur_top in enumerate(top): + # if peer score is greater than any set peer, overwrite + if unpack(self.PACK_FORMAT, cur_top[1])[0] < \ + unpack(self.PACK_FORMAT, peer_data)[0]: + overwrite = count + break # below else won't execute, so it will be overwritten + else: + # if not overwriting, go to next peer + continue + top[overwrite] = (peer, peer_data) + return top + def add_score(self, peer, new_score): shrunk = self._shrink_peer_address(peer) score, _ = unpack("qQ", self.db.get(shrunk)) @@ -61,7 +89,7 @@ class Peers: def update_seen(self, peer): peer = self._shrink_peer_address(peer) - _pack_and_store_info(peer, new_seen=int(time())) - - + self._pack_and_store_info(peer, new_seen=int(time())) + def add_peer(self, peer): + self._pack_and_store_info(peer) diff --git a/static-data/default-plugins/torgossip/runtest.py b/static-data/default-plugins/torgossip/runtest.py index 2b9256f7..e89eb6e3 100644 --- a/static-data/default-plugins/torgossip/runtest.py +++ b/static-data/default-plugins/torgossip/runtest.py @@ -13,16 +13,17 @@ def torgossip_runtest(test_manager): s_file = identifyhome.identify_home() + "/torgossip.sock" bl_test = blockcreator.create_anonvdf_block(b"test", "txt", 10) + shared_state = test_manager._too_many - #test_manager._too_many.get_by_string("PassToSafeDB").queue_then_store(b"test", "txt", 10) + #shared_state.get_by_string("PassToSafeDB").queue_then_store(b"test", "txt", 10) bl = subprocgenerate.vdf_block(b"test", "txt", 100) - blockio.store_block(bl, test_manager._too_many.get_by_string("SafeDB")) + blockio.store_block(bl, shared_state.get_by_string("SafeDB")) tsts = b'' for i in range(3): bl = subprocgenerate.vdf_block(b"test" + os.urandom(3), "tst", 100) tsts += bl.id - blockio.store_block(bl, test_manager._too_many.get_by_string("SafeDB")) + blockio.store_block(bl, shared_state.get_by_string("SafeDB")) bl_new = blockcreator.create_anonvdf_block(b"test5", "txt", 10) @@ -55,13 +56,17 @@ def torgossip_runtest(test_manager): s.sendall(b'5' + bl_new.id) assert s.recv(64) == bl_new.get_packed() - # test block was uploaded by getting it - s.sendall(b'7') - assert s.recv(64) == b"BYE" - s.sendall(b'41tst') assert s.recv(1000) == tsts[64:] s.sendall(b'42tst') assert s.recv(1000) == tsts[64*2:] + # test peer list + + shared_state.get_by_string('TorGossipPeers').add_peer() + + s.sendall(b'9') + assert s.recv(64) == b"BYE" + + diff --git a/tests/benchmarks/benchmark-neighbor-closeness.py b/tests/benchmarks/benchmark-neighbor-closeness.py deleted file mode 100644 index 25b5ecc1..00000000 --- a/tests/benchmarks/benchmark-neighbor-closeness.py +++ /dev/null @@ -1,34 +0,0 @@ -import sys, os -sys.path.append(".") -sys.path.append("src/") -import unittest, uuid -import subprocess -import time - -TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/' -print("Test directory:", TEST_DIR) -os.environ["ONIONR_HOME"] = TEST_DIR -from utils import createdirs, identifyhome -from streamfill import identify_neighbors - -onions = [] -p = subprocess.Popen(["scripts/generate-onions.py", '50000'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) -for line in iter(p.stdout.readline, b''): - line = line.decode() - onions.append(line.strip()) -p.terminate() - -p = subprocess.Popen(["scripts/generate-onions.py", '1'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) -for line in iter(p.stdout.readline, b''): - us = line.decode().strip() -p.terminate() - -start = time.time() -identify_neighbors(us, onions, 5) -print(time.time() - start) - - diff --git a/tests/test_streamfill_extract_ed25519.py b/tests/test_streamfill_extract_ed25519.py deleted file mode 100644 index c6dd2d94..00000000 --- a/tests/test_streamfill_extract_ed25519.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 -import sys, os -sys.path.append(".") -sys.path.append("src/") -import uuid -import binascii -from base64 import b32decode -TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/' -print("Test directory:", TEST_DIR) -os.environ["ONIONR_HOME"] = TEST_DIR -import unittest - -from streamfill import extract_ed25519_from_onion_address - -class TestStreamfillExtractEd25519(unittest.TestCase): - def test_extract_normal_onion(self): - hardcodedCorrect = b'y\xbc\xc6%\x18K\x05\x19Iu\xc2\x8bf\xb6k\x04i\xf7\xf6Uo\xb1\xac1\x89\xa7\x9b@\xdd\xa3/\x1f' - correct = b32decode(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd'.upper())[:-3] - self.assertEqual(correct, hardcodedCorrect) - self.assertEqual(correct, extract_ed25519_from_onion_address('pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion')) - def test_extract_normal_onion_bytes(self): - hardcodedCorrect = b'y\xbc\xc6%\x18K\x05\x19Iu\xc2\x8bf\xb6k\x04i\xf7\xf6Uo\xb1\xac1\x89\xa7\x9b@\xdd\xa3/\x1f' - correct = b32decode(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd'.upper())[:-3] - self.assertEqual(correct, hardcodedCorrect) - self.assertEqual(correct, extract_ed25519_from_onion_address(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion')) - def test_extract_no_ext(self): - hardcodedCorrect = b'y\xbc\xc6%\x18K\x05\x19Iu\xc2\x8bf\xb6k\x04i\xf7\xf6Uo\xb1\xac1\x89\xa7\x9b@\xdd\xa3/\x1f' - correct = b32decode(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd'.upper())[:-3] - self.assertEqual(correct, hardcodedCorrect) - self.assertEqual(correct, extract_ed25519_from_onion_address('pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd')) - def test_extract_no_ext_bytes(self): - hardcodedCorrect = b'y\xbc\xc6%\x18K\x05\x19Iu\xc2\x8bf\xb6k\x04i\xf7\xf6Uo\xb1\xac1\x89\xa7\x9b@\xdd\xa3/\x1f' - correct = b32decode(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd'.upper())[:-3] - self.assertEqual(correct, hardcodedCorrect) - self.assertEqual(correct, extract_ed25519_from_onion_address(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd')) - def test_fail_length(self): - hardcodedCorrect = b'y\xbc\xc6%\x18K\x05\x19Iu\xc2\x8bf\xb6k\x04i\xf7\xf6Uo\xb1\xac1\x89\xa7\x9b@\xdd\xa3/\x1f' - correct = b32decode(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd'.upper())[:-3] - self.assertEqual(correct, hardcodedCorrect) - try: - extract_ed25519_from_onion_address(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscry') - except binascii.Error: - pass - def test_fail_length_onion(self): - hardcodedCorrect = b'y\xbc\xc6%\x18K\x05\x19Iu\xc2\x8bf\xb6k\x04i\xf7\xf6Uo\xb1\xac1\x89\xa7\x9b@\xdd\xa3/\x1f' - correct = b32decode(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd'.upper())[:-3] - self.assertEqual(correct, hardcodedCorrect) - try: - extract_ed25519_from_onion_address(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscry.onion') - except binascii.Error: - pass - -unittest.main() diff --git a/tests/test_streamfill_neighbors.py b/tests/test_streamfill_neighbors.py deleted file mode 100644 index 8471fa53..00000000 --- a/tests/test_streamfill_neighbors.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python3 -import sys, os -import subprocess -sys.path.append(".") -sys.path.append("src/") -import uuid -import binascii -from base64 import b32decode -TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/' -print("Test directory:", TEST_DIR) -os.environ["ONIONR_HOME"] = TEST_DIR -import unittest - -from streamfill import identify_neighbors, extract_ed25519_from_onion_address - -class TestStreamfillNeighbors(unittest.TestCase): - def test_neighbor_closeness_consistent(self): - main = '7uifxsgidchopmdwmtip6x4ydra6cpf2ov4ghj2lzx5uydyssduh5qid.onion' - others = ['bxxajpimlonmbxb5jzjre3go3dvfobqyayqwpksd6zpjz4s4mknstwyd.onion', '2zofaifd6s3flwbv5wl4vtgnesbprc4f2ptljl4a47dfkvrbmw3e5iqd.onion', '6umslj7jtzu27n4jgf3byn55ztz5mkoqocx32zwjya6rbnxqjpyysyyd.onion'] - main_num = int.from_bytes(extract_ed25519_from_onion_address(main), 'big') - - test_data = identify_neighbors(main, others, 3) - - my_result = [] - for i in others: - my_result.append((i, abs(main_num - int.from_bytes(extract_ed25519_from_onion_address(i), 'big')))) - my_result.sort(key=lambda p: p[1]) - - final = [] - for i in my_result: - final.append(i[0]) - self.assertListEqual(final, test_data) - - - def test_neighbor_closeness_random(self): - onions = [] - p = subprocess.Popen(["scripts/generate-onions.py", '100'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - for line in iter(p.stdout.readline, b''): - line = line.decode() - onions.append(line.strip()) - p.terminate() - main = '7uifxsgidchopmdwmtip6x4ydra6cpf2ov4ghj2lzx5uydyssduh5qid.onion' - main_num = int.from_bytes(extract_ed25519_from_onion_address(main), 'big') - - test_data = identify_neighbors(main, onions, 100) - - my_result = [] - for i in onions: - my_result.append((i, abs(main_num - int.from_bytes(extract_ed25519_from_onion_address(i), 'big')))) - my_result.sort(key=lambda p: p[1]) - - final = [] - for i in my_result: - final.append(i[0]) - self.assertListEqual(final, test_data) - - -unittest.main()