work on torgossip

scripts/generate-onions.py

@@ -2,40 +2,21 @@
 
 import sys
 import os
-import stem
+from base64 import b32encode
-from stem import process
+from hashlib import sha3_256
-from stem.control import Controller
-if not os.path.exists('onionr.sh'):
-    os.chdir('../')
-sys.path.append("src/")
 
 try:
-    sys.argv[1]
+  amount = int(sys.argv[1])
 except IndexError:
-    sys.exit(1)
+  amount = 1
 
-tor_process = process.launch_tor_with_config(
+version = int(3).to_bytes(1, "little")
-    completion_percent=0,
+for i in range(amount):
-  config = {
+  pubkey = os.urandom(32)
-    'ControlPort': '2778',
+  #digest = sha3_256(b".onion checksum" + pubkey + version).digest()[:2]
-    'DisableNetwork': '1',
+  digest = sha3_256()
-    'Log': [
+  digest.update(b".onion checksum")
-      'NOTICE stdout',
+  digest.update(pubkey)
-      'ERR file /tmp/tor_error_log',
+  digest.update(version)
-    ],
+  digest = digest.digest()[:2]
-  },
+  print(b32encode(pubkey + digest + version).decode().lower() + ".onion")
-)
-
-with Controller.from_port('127.0.0.1', 2778) as controller:
-    controller.authenticate()
-    for i in range(1024, 1024 + int(sys.argv[1])):
-        hs = controller.create_ephemeral_hidden_service(
-            {80: i},
-            key_type='NEW',
-            key_content='ED25519-V3',
-            await_publication=False,
-            detached=True)
-        print(hs.service_id + ".onion")
-        controller.remove_ephemeral_hidden_service(hs.service_id)
-
-tor_process.kill()

static-data/base64-code-whitelist.txt

@@ -67,3 +67,4 @@ ZABkAWwAWgBkAGQBbAFaAWQAZAJsAm0DWgMBAGQAZANsBG0FWgUBAGQAZARsBm0HWgcBAGQFZAaEAFoI
 ZABkAWwAWgBkAGQBbAFaAWQAZAJsAm0DWgMBAGQAZANsBG0FWgUBAGQAZARsBm0HWgcBAGQAZAVsCG0JWgkBAGQAZAZsCm0LWgsBAGQHZAiEAFoMZAFTAA==
 ZABkAWwAWgBkAGQBbAFaAWQAZAJsAm0DWgMBAGQAZANsBG0FWgUBAGQAZARsBm0HWgcBAGQAZAVsCG0JWgkBAGQAZAZsCm0LWgsBAGQAZAFsCFoIZAdkCIQAWgxkAVMA
 ZABaAGQBZAJsAW0CWgIBAGQBZANsA20EWgQBAGQBZARsBVoFZAFkBGwBWgFkAWQFbAZtB1oHAQBkBmQHhABaCGUJZAicAWQJZAqEBFoKZQlkCJwBZAtkDIQEWgtkDWQOhABaDGQEUwA=
+ZABaAGQBZAJsAW0CWgIBAGQBZANsA20EWgQBAGQBZARsBW0GWgYBAGQBZAVsB1oHZAFkBWwDWgNkAWQGbAhtCVoJAQBkB2QIhABaCmULZAmcAWQKZAuEBFoMZAxkDYQAWg1lC2QJnAFkDmQPhARaDmQQZBGEAFoPZAVTAA==

static-data/default-plugins/torgossip/client.py

@@ -21,5 +21,6 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 
 def client_pool(shared_state):
+    return
+
 
-    

static-data/default-plugins/torgossip/commandhandlers.py

@@ -9,6 +9,7 @@ import blockio
 
 import onionrblocks
 from kasten import Kasten
+
 """
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -25,6 +26,12 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 """
 
 
+def peer_exchange(peers: 'Peers', num_of_peers: bytes):
+    #7
+    num_of_peers = int.from_bytes(num_of_peers, 'little')
+    return peers.get_highest_score_peers(num_of_peers)
+
+
 def put_block(safe_db, block):
     #6
     block_hash = block[:64]

static-data/default-plugins/torgossip/commands.py

@@ -28,4 +28,5 @@ class GossipCommands(IntEnum):
     GET_BLOCK = 5,
     PUT_BLOCK = 6,
     PEER_EXCHANGE = 7,
-    EXIT = 8
+    ANNOUNCE_PEER = 8,
+    EXIT = 9

static-data/default-plugins/torgossip/main.py

@@ -24,12 +24,16 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <https://www.gnu.org/licenses/>.
 """
 plugin_name = 'torgossip'
+
 from server import start_server
+from peerdb import Peers
 from runtest import torgossip_runtest
 
 def on_init(api, data=None):
     shared_state = data
 
+    shared_state.get(Peers)
+
     shared_state.get_by_string(
         "OnionrRunTestManager").plugin_tests.append(torgossip_runtest)
 

static-data/default-plugins/torgossip/peerdb.py

@@ -26,11 +26,11 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 
 
-class Peers:
+class TorGossipPeers:  # name it this way to avoid collisions in SharedState
 
     PACK_FORMAT = "qQ"
 
-    def __init__(self, encrypt: bool):
+    def __init__(self, encrypt=True):
         self.db = safedb.SafeDB(
             identify_home() + "/torgossip-peers.db", protected=encrypt)
 
@@ -42,8 +42,11 @@ class Peers:
 
     def _pack_and_store_info(self, peer, new_score=None, new_seen=None):
         # Repack peer information with new value(s) and store it
-        score, seen = unpack(self.PACK_FORMAT, self.db.get(
+        peer = self._shrink_peer_address(peer)
-            self._shrink_peer_address(peer)))
+        try:
+            score, seen = unpack(self.PACK_FORMAT, self.db.get(peer))
+        except KeyError:
+            score = seen = 0
 
         if new_score:
             score = new_score
@@ -52,6 +55,31 @@ class Peers:
 
         self.db.put(peer, pack(self.PACK_FORMAT, score, seen))
 
+    def get_highest_score_peers(self, max=5):
+        assert max >= 1
+        peer = self.db.db_conn.firstkey()
+
+        top = [(peer, self.db.get(peer))]
+
+        for peer in self.db.db_conn.nextkey(peer):
+            peer_data = self.db.get(peer)
+            overwrite = None
+
+            if len(top) != max:
+                top.append((peer, peer_data))
+                continue
+            for count, cur_top in enumerate(top):
+                # if peer score is greater than any set peer, overwrite
+                if unpack(self.PACK_FORMAT, cur_top[1])[0] < \
+                        unpack(self.PACK_FORMAT, peer_data)[0]:
+                    overwrite = count
+                    break  # below else won't execute, so it will be overwritten
+            else:
+                # if not overwriting, go to next peer
+                continue
+            top[overwrite] = (peer, peer_data)
+        return top
+
     def add_score(self, peer, new_score):
         shrunk = self._shrink_peer_address(peer)
         score, _ = unpack("qQ", self.db.get(shrunk))
@@ -61,7 +89,7 @@ class Peers:
 
     def update_seen(self, peer):
         peer = self._shrink_peer_address(peer)
-        _pack_and_store_info(peer, new_seen=int(time()))
+        self._pack_and_store_info(peer, new_seen=int(time()))
-
-
 
+    def add_peer(self, peer):
+        self._pack_and_store_info(peer)

static-data/default-plugins/torgossip/runtest.py

@@ -13,16 +13,17 @@ def torgossip_runtest(test_manager):
 
     s_file = identifyhome.identify_home() + "/torgossip.sock"
     bl_test = blockcreator.create_anonvdf_block(b"test", "txt", 10)
+    shared_state = test_manager._too_many
 
-    #test_manager._too_many.get_by_string("PassToSafeDB").queue_then_store(b"test", "txt", 10)
+    #shared_state.get_by_string("PassToSafeDB").queue_then_store(b"test", "txt", 10)
     bl = subprocgenerate.vdf_block(b"test", "txt", 100)
-    blockio.store_block(bl, test_manager._too_many.get_by_string("SafeDB"))
+    blockio.store_block(bl, shared_state.get_by_string("SafeDB"))
 
     tsts = b''
     for i in range(3):
         bl = subprocgenerate.vdf_block(b"test" + os.urandom(3), "tst", 100)
         tsts += bl.id
-        blockio.store_block(bl, test_manager._too_many.get_by_string("SafeDB"))
+        blockio.store_block(bl, shared_state.get_by_string("SafeDB"))
 
     bl_new = blockcreator.create_anonvdf_block(b"test5", "txt", 10)
 
@@ -55,13 +56,17 @@ def torgossip_runtest(test_manager):
         s.sendall(b'5' + bl_new.id)
         assert s.recv(64) == bl_new.get_packed()
 
-        # test block was uploaded by getting it
-        s.sendall(b'7')
-        assert s.recv(64) == b"BYE"
-
         s.sendall(b'41tst')
         assert s.recv(1000) == tsts[64:]
 
         s.sendall(b'42tst')
         assert s.recv(1000) == tsts[64*2:]
 
+        # test peer list
+        
+        shared_state.get_by_string('TorGossipPeers').add_peer()
+
+        s.sendall(b'9')
+        assert s.recv(64) == b"BYE"
+
+

tests/benchmarks/benchmark-neighbor-closeness.py

@@ -1,34 +0,0 @@
-import sys, os
-sys.path.append(".")
-sys.path.append("src/")
-import unittest, uuid
-import subprocess
-import time
-
-TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
-print("Test directory:", TEST_DIR)
-os.environ["ONIONR_HOME"] = TEST_DIR
-from utils import createdirs, identifyhome
-from streamfill import identify_neighbors
-
-onions = []
-p = subprocess.Popen(["scripts/generate-onions.py", '50000'],
-                        stdout=subprocess.PIPE,
-                        stderr=subprocess.PIPE)
-for line in iter(p.stdout.readline, b''):
-    line = line.decode()
-    onions.append(line.strip())
-p.terminate()
-
-p = subprocess.Popen(["scripts/generate-onions.py", '1'],
-                        stdout=subprocess.PIPE,
-                        stderr=subprocess.PIPE)
-for line in iter(p.stdout.readline, b''):
-    us = line.decode().strip()
-p.terminate()
-
-start = time.time()
-identify_neighbors(us, onions, 5)
-print(time.time() - start)
-
-

tests/test_streamfill_extract_ed25519.py

@@ -1,53 +0,0 @@
-#!/usr/bin/env python3
-import sys, os
-sys.path.append(".")
-sys.path.append("src/")
-import uuid
-import binascii
-from base64 import b32decode
-TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
-print("Test directory:", TEST_DIR)
-os.environ["ONIONR_HOME"] = TEST_DIR
-import unittest
-
-from streamfill import extract_ed25519_from_onion_address
-
-class TestStreamfillExtractEd25519(unittest.TestCase):
-    def test_extract_normal_onion(self):
-        hardcodedCorrect = b'y\xbc\xc6%\x18K\x05\x19Iu\xc2\x8bf\xb6k\x04i\xf7\xf6Uo\xb1\xac1\x89\xa7\x9b@\xdd\xa3/\x1f'
-        correct = b32decode(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd'.upper())[:-3]
-        self.assertEqual(correct, hardcodedCorrect)
-        self.assertEqual(correct, extract_ed25519_from_onion_address('pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion'))
-    def test_extract_normal_onion_bytes(self):
-        hardcodedCorrect = b'y\xbc\xc6%\x18K\x05\x19Iu\xc2\x8bf\xb6k\x04i\xf7\xf6Uo\xb1\xac1\x89\xa7\x9b@\xdd\xa3/\x1f'
-        correct = b32decode(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd'.upper())[:-3]
-        self.assertEqual(correct, hardcodedCorrect)
-        self.assertEqual(correct, extract_ed25519_from_onion_address(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion'))
-    def test_extract_no_ext(self):
-        hardcodedCorrect = b'y\xbc\xc6%\x18K\x05\x19Iu\xc2\x8bf\xb6k\x04i\xf7\xf6Uo\xb1\xac1\x89\xa7\x9b@\xdd\xa3/\x1f'
-        correct = b32decode(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd'.upper())[:-3]
-        self.assertEqual(correct, hardcodedCorrect)
-        self.assertEqual(correct, extract_ed25519_from_onion_address('pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd'))
-    def test_extract_no_ext_bytes(self):
-        hardcodedCorrect = b'y\xbc\xc6%\x18K\x05\x19Iu\xc2\x8bf\xb6k\x04i\xf7\xf6Uo\xb1\xac1\x89\xa7\x9b@\xdd\xa3/\x1f'
-        correct = b32decode(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd'.upper())[:-3]
-        self.assertEqual(correct, hardcodedCorrect)
-        self.assertEqual(correct, extract_ed25519_from_onion_address(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd'))
-    def test_fail_length(self):
-        hardcodedCorrect = b'y\xbc\xc6%\x18K\x05\x19Iu\xc2\x8bf\xb6k\x04i\xf7\xf6Uo\xb1\xac1\x89\xa7\x9b@\xdd\xa3/\x1f'
-        correct = b32decode(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd'.upper())[:-3]
-        self.assertEqual(correct, hardcodedCorrect)
-        try:
-            extract_ed25519_from_onion_address(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscry')
-        except binascii.Error:
-            pass
-    def test_fail_length_onion(self):
-        hardcodedCorrect = b'y\xbc\xc6%\x18K\x05\x19Iu\xc2\x8bf\xb6k\x04i\xf7\xf6Uo\xb1\xac1\x89\xa7\x9b@\xdd\xa3/\x1f'
-        correct = b32decode(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd'.upper())[:-3]
-        self.assertEqual(correct, hardcodedCorrect)
-        try:
-            extract_ed25519_from_onion_address(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscry.onion')
-        except binascii.Error:
-            pass
-
-unittest.main()

tests/test_streamfill_neighbors.py

@@ -1,60 +0,0 @@
-#!/usr/bin/env python3
-import sys, os
-import subprocess
-sys.path.append(".")
-sys.path.append("src/")
-import uuid
-import binascii
-from base64 import b32decode
-TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
-print("Test directory:", TEST_DIR)
-os.environ["ONIONR_HOME"] = TEST_DIR
-import unittest
-
-from streamfill import identify_neighbors, extract_ed25519_from_onion_address
-
-class TestStreamfillNeighbors(unittest.TestCase):
-    def test_neighbor_closeness_consistent(self):
-        main = '7uifxsgidchopmdwmtip6x4ydra6cpf2ov4ghj2lzx5uydyssduh5qid.onion'
-        others = ['bxxajpimlonmbxb5jzjre3go3dvfobqyayqwpksd6zpjz4s4mknstwyd.onion', '2zofaifd6s3flwbv5wl4vtgnesbprc4f2ptljl4a47dfkvrbmw3e5iqd.onion', '6umslj7jtzu27n4jgf3byn55ztz5mkoqocx32zwjya6rbnxqjpyysyyd.onion']
-        main_num = int.from_bytes(extract_ed25519_from_onion_address(main), 'big')
-
-        test_data = identify_neighbors(main, others, 3)
-
-        my_result = []
-        for i in others:
-            my_result.append((i, abs(main_num - int.from_bytes(extract_ed25519_from_onion_address(i), 'big'))))
-        my_result.sort(key=lambda p: p[1])
-
-        final = []
-        for i in my_result:
-            final.append(i[0])
-        self.assertListEqual(final, test_data)
-
-
-    def test_neighbor_closeness_random(self):
-        onions = []
-        p = subprocess.Popen(["scripts/generate-onions.py", '100'],
-                             stdout=subprocess.PIPE,
-                             stderr=subprocess.PIPE)
-        for line in iter(p.stdout.readline, b''):
-            line = line.decode()
-            onions.append(line.strip())
-        p.terminate()
-        main = '7uifxsgidchopmdwmtip6x4ydra6cpf2ov4ghj2lzx5uydyssduh5qid.onion'
-        main_num = int.from_bytes(extract_ed25519_from_onion_address(main), 'big')
-
-        test_data = identify_neighbors(main, onions, 100)
-
-        my_result = []
-        for i in onions:
-            my_result.append((i, abs(main_num - int.from_bytes(extract_ed25519_from_onion_address(i), 'big'))))
-        my_result.sort(key=lambda p: p[1])
-
-        final = []
-        for i in my_result:
-            final.append(i[0])
-        self.assertListEqual(final, test_data)
-
-
-unittest.main()