kev
/
Onionr
1
0
Fork 0
Code Issues 24 Pull Requests Projects Releases 2 Wiki Activity

work on torgossip

This commit is contained in:
Kevin Froman 2021-02-02 23:24:35 +00:00
parent 9d8c7a7224
commit 5ea90acd3f
11 changed files with 77 additions and 196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
scripts/generate-onions.py
View File

@ -2,40 +2,21 @@
import sys
import os
import stem
from stem import process
from stem.control import Controller
if not os.path.exists('onionr.sh'):
os.chdir('../')
sys.path.append("src/")
from base64 import b32encode
from hashlib import sha3_256
try:
sys.argv[1]
amount = int(sys.argv[1])
except IndexError:
sys.exit(1)
amount = 1
tor_process = process.launch_tor_with_config(
completion_percent=0,
config = {
'ControlPort': '2778',
'DisableNetwork': '1',
'Log': [
'NOTICE stdout',
'ERR file /tmp/tor_error_log',
],
},
)
with Controller.from_port('127.0.0.1', 2778) as controller:
controller.authenticate()
for i in range(1024, 1024 + int(sys.argv[1])):
hs = controller.create_ephemeral_hidden_service(
{80: i},
key_type='NEW',
key_content='ED25519-V3',
await_publication=False,
detached=True)
print(hs.service_id + ".onion")
controller.remove_ephemeral_hidden_service(hs.service_id)
tor_process.kill()
version = int(3).to_bytes(1, "little")
for i in range(amount):
pubkey = os.urandom(32)
#digest = sha3_256(b".onion checksum" + pubkey + version).digest()[:2]
digest = sha3_256()
digest.update(b".onion checksum")
digest.update(pubkey)
digest.update(version)
digest = digest.digest()[:2]
print(b32encode(pubkey + digest + version).decode().lower() + ".onion")

1
static-data/base64-code-whitelist.txt
View File

@ -67,3 +67,4 @@ ZABkAWwAWgBkAGQBbAFaAWQAZAJsAm0DWgMBAGQAZANsBG0FWgUBAGQAZARsBm0HWgcBAGQFZAaEAFoI
ZABkAWwAWgBkAGQBbAFaAWQAZAJsAm0DWgMBAGQAZANsBG0FWgUBAGQAZARsBm0HWgcBAGQAZAVsCG0JWgkBAGQAZAZsCm0LWgsBAGQHZAiEAFoMZAFTAA==
ZABkAWwAWgBkAGQBbAFaAWQAZAJsAm0DWgMBAGQAZANsBG0FWgUBAGQAZARsBm0HWgcBAGQAZAVsCG0JWgkBAGQAZAZsCm0LWgsBAGQAZAFsCFoIZAdkCIQAWgxkAVMA
ZABaAGQBZAJsAW0CWgIBAGQBZANsA20EWgQBAGQBZARsBVoFZAFkBGwBWgFkAWQFbAZtB1oHAQBkBmQHhABaCGUJZAicAWQJZAqEBFoKZQlkCJwBZAtkDIQEWgtkDWQOhABaDGQEUwA=
ZABaAGQBZAJsAW0CWgIBAGQBZANsA20EWgQBAGQBZARsBW0GWgYBAGQBZAVsB1oHZAFkBWwDWgNkAWQGbAhtCVoJAQBkB2QIhABaCmULZAmcAWQKZAuEBFoMZAxkDYQAWg1lC2QJnAFkDmQPhARaDmQQZBGEAFoPZAVTAA==

3
static-data/default-plugins/torgossip/client.py
View File

@ -21,5 +21,6 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
def client_pool(shared_state):
return

7
static-data/default-plugins/torgossip/commandhandlers.py
View File

@ -9,6 +9,7 @@ import blockio
import onionrblocks
from kasten import Kasten
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -25,6 +26,12 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def peer_exchange(peers: 'Peers', num_of_peers: bytes):
#7
num_of_peers = int.from_bytes(num_of_peers, 'little')
return peers.get_highest_score_peers(num_of_peers)
def put_block(safe_db, block):
#6
block_hash = block[:64]

3
static-data/default-plugins/torgossip/commands.py
View File

@ -28,4 +28,5 @@ class GossipCommands(IntEnum):
GET_BLOCK = 5,
PUT_BLOCK = 6,
PEER_EXCHANGE = 7,
EXIT = 8
ANNOUNCE_PEER = 8,
EXIT = 9

4
static-data/default-plugins/torgossip/main.py
View File

@ -24,12 +24,16 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
plugin_name = 'torgossip'
from server import start_server
from peerdb import Peers
from runtest import torgossip_runtest
def on_init(api, data=None):
shared_state = data
shared_state.get(Peers)
shared_state.get_by_string(
"OnionrRunTestManager").plugin_tests.append(torgossip_runtest)

42
static-data/default-plugins/torgossip/peerdb.py
View File

@ -26,11 +26,11 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
class Peers:
class TorGossipPeers: # name it this way to avoid collisions in SharedState
PACK_FORMAT = "qQ"
def __init__(self, encrypt: bool):
def __init__(self, encrypt=True):
self.db = safedb.SafeDB(
identify_home() + "/torgossip-peers.db", protected=encrypt)
@ -42,8 +42,11 @@ class Peers:
def _pack_and_store_info(self, peer, new_score=None, new_seen=None):
# Repack peer information with new value(s) and store it
score, seen = unpack(self.PACK_FORMAT, self.db.get(
self._shrink_peer_address(peer)))
peer = self._shrink_peer_address(peer)
try:
score, seen = unpack(self.PACK_FORMAT, self.db.get(peer))
except KeyError:
score = seen = 0
if new_score:
score = new_score
@ -52,6 +55,31 @@ class Peers:
self.db.put(peer, pack(self.PACK_FORMAT, score, seen))
def get_highest_score_peers(self, max=5):
assert max >= 1
peer = self.db.db_conn.firstkey()
top = [(peer, self.db.get(peer))]
for peer in self.db.db_conn.nextkey(peer):
peer_data = self.db.get(peer)
overwrite = None
if len(top) != max:
top.append((peer, peer_data))
continue
for count, cur_top in enumerate(top):
# if peer score is greater than any set peer, overwrite
if unpack(self.PACK_FORMAT, cur_top[1])[0] < \
unpack(self.PACK_FORMAT, peer_data)[0]:
overwrite = count
break # below else won't execute, so it will be overwritten
else:
# if not overwriting, go to next peer
continue
top[overwrite] = (peer, peer_data)
return top
def add_score(self, peer, new_score):
shrunk = self._shrink_peer_address(peer)
score, _ = unpack("qQ", self.db.get(shrunk))
@ -61,7 +89,7 @@ class Peers:
def update_seen(self, peer):
peer = self._shrink_peer_address(peer)
_pack_and_store_info(peer, new_seen=int(time()))
self._pack_and_store_info(peer, new_seen=int(time()))
def add_peer(self, peer):
self._pack_and_store_info(peer)

19
static-data/default-plugins/torgossip/runtest.py
View File

@ -13,16 +13,17 @@ def torgossip_runtest(test_manager):
s_file = identifyhome.identify_home() + "/torgossip.sock"
bl_test = blockcreator.create_anonvdf_block(b"test", "txt", 10)
shared_state = test_manager._too_many
#test_manager._too_many.get_by_string("PassToSafeDB").queue_then_store(b"test", "txt", 10)
#shared_state.get_by_string("PassToSafeDB").queue_then_store(b"test", "txt", 10)
bl = subprocgenerate.vdf_block(b"test", "txt", 100)
blockio.store_block(bl, test_manager._too_many.get_by_string("SafeDB"))
blockio.store_block(bl, shared_state.get_by_string("SafeDB"))
tsts = b''
for i in range(3):
bl = subprocgenerate.vdf_block(b"test" + os.urandom(3), "tst", 100)
tsts += bl.id
blockio.store_block(bl, test_manager._too_many.get_by_string("SafeDB"))
blockio.store_block(bl, shared_state.get_by_string("SafeDB"))
bl_new = blockcreator.create_anonvdf_block(b"test5", "txt", 10)
@ -55,13 +56,17 @@ def torgossip_runtest(test_manager):
s.sendall(b'5' + bl_new.id)
assert s.recv(64) == bl_new.get_packed()
# test block was uploaded by getting it
s.sendall(b'7')
assert s.recv(64) == b"BYE"
s.sendall(b'41tst')
assert s.recv(1000) == tsts[64:]
s.sendall(b'42tst')
assert s.recv(1000) == tsts[64*2:]
# test peer list
shared_state.get_by_string('TorGossipPeers').add_peer()
s.sendall(b'9')
assert s.recv(64) == b"BYE"

34
tests/benchmarks/benchmark-neighbor-closeness.py
View File

@ -1,34 +0,0 @@
import sys, os
sys.path.append(".")
sys.path.append("src/")
import unittest, uuid
import subprocess
import time
TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
print("Test directory:", TEST_DIR)
os.environ["ONIONR_HOME"] = TEST_DIR
from utils import createdirs, identifyhome
from streamfill import identify_neighbors
onions = []
p = subprocess.Popen(["scripts/generate-onions.py", '50000'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
for line in iter(p.stdout.readline, b''):
line = line.decode()
onions.append(line.strip())
p.terminate()
p = subprocess.Popen(["scripts/generate-onions.py", '1'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
for line in iter(p.stdout.readline, b''):
us = line.decode().strip()
p.terminate()
start = time.time()
identify_neighbors(us, onions, 5)
print(time.time() - start)

53
tests/test_streamfill_extract_ed25519.py
View File

@ -1,53 +0,0 @@
#!/usr/bin/env python3
import sys, os
sys.path.append(".")
sys.path.append("src/")
import uuid
import binascii
from base64 import b32decode
TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
print("Test directory:", TEST_DIR)
os.environ["ONIONR_HOME"] = TEST_DIR
import unittest
from streamfill import extract_ed25519_from_onion_address
class TestStreamfillExtractEd25519(unittest.TestCase):
def test_extract_normal_onion(self):
hardcodedCorrect = b'y\xbc\xc6%\x18K\x05\x19Iu\xc2\x8bf\xb6k\x04i\xf7\xf6Uo\xb1\xac1\x89\xa7\x9b@\xdd\xa3/\x1f'
correct = b32decode(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd'.upper())[:-3]
self.assertEqual(correct, hardcodedCorrect)
self.assertEqual(correct, extract_ed25519_from_onion_address('pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion'))
def test_extract_normal_onion_bytes(self):
hardcodedCorrect = b'y\xbc\xc6%\x18K\x05\x19Iu\xc2\x8bf\xb6k\x04i\xf7\xf6Uo\xb1\xac1\x89\xa7\x9b@\xdd\xa3/\x1f'
correct = b32decode(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd'.upper())[:-3]
self.assertEqual(correct, hardcodedCorrect)
self.assertEqual(correct, extract_ed25519_from_onion_address(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion'))
def test_extract_no_ext(self):
hardcodedCorrect = b'y\xbc\xc6%\x18K\x05\x19Iu\xc2\x8bf\xb6k\x04i\xf7\xf6Uo\xb1\xac1\x89\xa7\x9b@\xdd\xa3/\x1f'
correct = b32decode(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd'.upper())[:-3]
self.assertEqual(correct, hardcodedCorrect)
self.assertEqual(correct, extract_ed25519_from_onion_address('pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd'))
def test_extract_no_ext_bytes(self):
hardcodedCorrect = b'y\xbc\xc6%\x18K\x05\x19Iu\xc2\x8bf\xb6k\x04i\xf7\xf6Uo\xb1\xac1\x89\xa7\x9b@\xdd\xa3/\x1f'
correct = b32decode(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd'.upper())[:-3]
self.assertEqual(correct, hardcodedCorrect)
self.assertEqual(correct, extract_ed25519_from_onion_address(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd'))
def test_fail_length(self):
hardcodedCorrect = b'y\xbc\xc6%\x18K\x05\x19Iu\xc2\x8bf\xb6k\x04i\xf7\xf6Uo\xb1\xac1\x89\xa7\x9b@\xdd\xa3/\x1f'
correct = b32decode(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd'.upper())[:-3]
self.assertEqual(correct, hardcodedCorrect)
try:
extract_ed25519_from_onion_address(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscry')
except binascii.Error:
pass
def test_fail_length_onion(self):
hardcodedCorrect = b'y\xbc\xc6%\x18K\x05\x19Iu\xc2\x8bf\xb6k\x04i\xf7\xf6Uo\xb1\xac1\x89\xa7\x9b@\xdd\xa3/\x1f'
correct = b32decode(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd'.upper())[:-3]
self.assertEqual(correct, hardcodedCorrect)
try:
extract_ed25519_from_onion_address(b'pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscry.onion')
except binascii.Error:
pass
unittest.main()

60
tests/test_streamfill_neighbors.py
View File

@ -1,60 +0,0 @@
#!/usr/bin/env python3
import sys, os
import subprocess
sys.path.append(".")
sys.path.append("src/")
import uuid
import binascii
from base64 import b32decode
TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
print("Test directory:", TEST_DIR)
os.environ["ONIONR_HOME"] = TEST_DIR
import unittest
from streamfill import identify_neighbors, extract_ed25519_from_onion_address
class TestStreamfillNeighbors(unittest.TestCase):
def test_neighbor_closeness_consistent(self):
main = '7uifxsgidchopmdwmtip6x4ydra6cpf2ov4ghj2lzx5uydyssduh5qid.onion'
others = ['bxxajpimlonmbxb5jzjre3go3dvfobqyayqwpksd6zpjz4s4mknstwyd.onion', '2zofaifd6s3flwbv5wl4vtgnesbprc4f2ptljl4a47dfkvrbmw3e5iqd.onion', '6umslj7jtzu27n4jgf3byn55ztz5mkoqocx32zwjya6rbnxqjpyysyyd.onion']
main_num = int.from_bytes(extract_ed25519_from_onion_address(main), 'big')
test_data = identify_neighbors(main, others, 3)
my_result = []
for i in others:
my_result.append((i, abs(main_num - int.from_bytes(extract_ed25519_from_onion_address(i), 'big'))))
my_result.sort(key=lambda p: p[1])
final = []
for i in my_result:
final.append(i[0])
self.assertListEqual(final, test_data)
def test_neighbor_closeness_random(self):
onions = []
p = subprocess.Popen(["scripts/generate-onions.py", '100'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
for line in iter(p.stdout.readline, b''):
line = line.decode()
onions.append(line.strip())
p.terminate()
main = '7uifxsgidchopmdwmtip6x4ydra6cpf2ov4ghj2lzx5uydyssduh5qid.onion'
main_num = int.from_bytes(extract_ed25519_from_onion_address(main), 'big')
test_data = identify_neighbors(main, onions, 100)
my_result = []
for i in onions:
my_result.append((i, abs(main_num - int.from_bytes(extract_ed25519_from_onion_address(i), 'big'))))
my_result.sort(key=lambda p: p[1])
final = []
for i in my_result:
final.append(i[0])
self.assertListEqual(final, test_data)
unittest.main()