work on implementing torgossip

src/bigbrother/ministry/ofcommunication.py

@@ -35,6 +35,9 @@ def detect_socket_leaks(socket_event):
     try:
         ip_address = ipaddress.ip_address(ip_address)
     except ValueError:
+        if ip_address == "/":
+            # unix socket
+            return
         logger.warn(f'Conn made to {ip_address} outside of Tor/similar')
         raise \
             NetworkLeak('Conn to host/non local IP, this is a privacy issue!')

src/blockcreatorqueue/__init__.py

@@ -28,6 +28,19 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 class AlreadyGenerating(Exception): pass  # noqa
 
 
+
+class PassToSafeDB:
+    def __init__(self, db: 'SafeDB'):
+        self.db = db
+        self.block_creator_queue = BlockCreatorQueue(self.store_kasten)
+
+    def store_kasten(self, kasten_object):
+        self.db.put(kasten_object.id, kasten_object.get_packed())
+
+    def queue_then_store(self, block_data, block_type, ttl, **block_metadata):
+        self.block_creator_queue.queue_block(block_data, block_type, ttl, **block_metadata)
+
+
 class BlockCreatorQueue:
     def __init__(
             self, callback_func: Callable, *additional_callback_func_args,
@@ -66,4 +79,3 @@ class BlockCreatorQueue:
 
         Thread(target=_do_create, daemon=True).start()
         return digest
-

src/httpapi/serializedapi/__init__.py

@@ -62,4 +62,3 @@ def serialized(name: str) -> Response:
         if isinstance(attr, int):
             attr = str(attr)
         return Response(attr, content_type='application/octet-stream')
-

src/lan/server/__init__.py

@@ -52,6 +52,8 @@ class LANServer:
 
         @app.before_request
         def dns_rebinding_prevention():
+            if not ipaddress.ip_address(request.remote_addr).is_private:
+                abort(403)
             if request.remote_addr in lan_ips or \
                     ipaddress.ip_address(request.remote_addr).is_loopback:
                 if time.time() - _start_time > 600:

src/netcontroller/torcontrol/__init__.py

@@ -93,6 +93,10 @@ class NetController:
                 if 'bootstrapped 100' in line.decode().lower():
                     logger.info(line.decode(), terminal=True)
                     break
+                elif 'asking for networkstatus consensus' in line.decode().lower():
+                    logger.warn(
+                        "Tor has to load consensus, this should be faster next time," +
+                        " unless Onionr data is deleted.", terminal=True)
                 elif 'opening socks listener' in line.decode().lower():
                     logger.debug(line.decode().replace('\n', ''))
                 else:

src/netcontroller/torcontrol/gentorrc.py

@@ -72,8 +72,6 @@ def generate_torrc(net_controller: 'NetController',
 DataDirectory """ + home_dir + """tordata/
 CookieAuthentication 1
 KeepalivePeriod 40
-SafeSocks 1
-TestSocks 1
 CircuitsAvailableTimeout 86400
 ControlPort """ + str(control_port) + """
 HashedControlPassword """ + str(password) + """

src/onionrcommands/daemonlaunch/__init__.py

@@ -44,6 +44,7 @@ from sneakernet import sneakernet_import_thread
 from onionrstatistics.devreporting import statistics_reporter
 from setupkvvars import setup_kv
 from communicatorutils.housekeeping import clean_blocks_not_meeting_pow
+from blockcreatorqueue import BlockCreatorQueue, PassToSafeDB
 from .spawndaemonthreads import spawn_client_threads
 from .loadsafedb import load_safe_db
 """
@@ -181,6 +182,7 @@ def daemon():
     shared_state.get(serializeddata.SerializedData)
 
     shared_state.add(load_safe_db(config))
+    shared_state.add(PassToSafeDB(shared_state.get_by_string('SafeDB')))
 
     shared_state.share_object()  # share the parent object to the threads
 

src/runtests/__init__.py

@@ -53,9 +53,11 @@ class OnionrRunTestManager:
     def __init__(self):
         self.success: bool = True
         self.run_date: int = 0
+        self.plugin_tests = []
 
     def run_tests(self):
         tests = list(RUN_TESTS)
+        tests.extend(self.plugin_tests)
         SystemRandom().shuffle(tests)
         cur_time = epoch.get_epoch()
         logger.info(f"Doing runtime tests at {cur_time}")

static-data/base64-code-whitelist.txt

@@ -44,3 +44,26 @@ ZABkAWwAbQFaAQEAZABkAmwCbQNaAwEAZQFkA2QEZAVnAoMCWgRlA2QGZQWDAloGZQNkB2UHgwJaCGUD
 ZABkAYQAWgBkAlMA
 ZABkAWwAWgBkAGQBbAFaAWQCZAOEAFoCZARkBYQAWgNkAVMA
 ZABkAWwAWgBlAWQCnAFkA2QEhARaAmQFZAaEAFoDZAFTAA==
+ZABkAWQCZANnBFoAZARkBWwBWgFkBGQFbAJaAmQEZAVsA1oDZARkBWwEWgRkBGQFbAVaBWQEZAVsBloGZARkBWwHWgdkBGQFbAhaCGQEZAVsCVoJZAZkB2wKbQtaCwEAZAZkCGwKbQxaDG0NWg0BAGQGZAlsDm0PWg8BAGUPahBaEXokZARkBWwSWhJkBGQKbBJtE1oTbRRaFG0VWhVtFloWAQBXAG4kBABlF2sKcsoBAAEAAQBlA2oYZAtrAnLCggBkBVoSWQBuAlgAZAxaGWQNWhplCKAboQBaHGQOWh1kDmcBWh5lH2UEZA+DAnL+ZA9aHWUeZA9nATcAWh5lA2oYZAtrApABchhkEFodZR5kEGcBNwBaHmUaZgFkEWQShAFaIGQTZBSEAFohZBVkFoQAWiJkF2QYhABaI2QZZBqEAFokRwBkG2QchABkHIMCWiVlEpABcmhHAGQdZB6EAGQeZSWDA1omRwBkH2QghABkIGUlgwNaJ0cAZCFkAYQAZAFlKIMDWilkT2QiZACEAVoqZQNqGGQLawOQAXKqZFBkJGQChAFaK24KZFFkJWQChAFaK0cAZCZkJ4QAZCdlKIMDWixkKGQphABaLWUDahhkC2sCkAFy8EcAZCpkK4QAZCtlKIMDWi5kLGQthABaL2QuWjBkL1oxZDBaMmQxWjNkMmQzhABaNGQ0ZDWEAFo1RwBkNmQ3hABkN2UogwNaNmQ4ZDmEAFo3ZDpkO4QAWjhHAGQ8ZD2EAGQ9ZSmDA1o5ZD5kP4QAWjplA2oYZAtrApACcnRkQGRBhABaO2USajxlEmo9aAJaPmRSZEJkA4QBWj9uLGQEZAVsQFpAZR9lQGRDgwKQAnKQZUBqQVpCbgZlQGpDWkJkU2REZAOEAVo/ZQNqGGQLawKQAnLmZEVkRoQAWkRkR2RIhABaRWUPoEZlJ2VEoQIBAGRJZEqEAFpHZEtkTIQAWkhlD6BGZSZlR6ECAQBuHGRNZEaEAFpEZE5kSIQAWkVlD6BGZSdlRKECAQBkBVMA
+ZABkAWwAWgBkAGQBbAFaAWQAZAFsAloCZABkAWwDWgNkAGQBbARaBGQAZAFsBVoFZABkAmwGbQdaBwEAZANkBGwIbQlaCQEAZAVkBmQHZAhkCWQKZAtkDGQNZA5kD2QQZBFkEmQTZw9aCmQAWgtkFFoMZBVaDWQWWg5kF1oPZBhaEGQZWhFkAWESZBphE2QbZAWEAFoUZBxkBoQAWhVkHWQHhABaFmQeZAiEAFoXZB9kCYQAWhhkQGQgZAqEAVoZZCFkIoQAWhpkI2QkhABaG2UagwBaHGQlZCaEAFodZCdkC4QAWh5lA6AfoQBaIGUBoCGhAFoiZChkKYQAWiNkKmQMhABaJGkAWiVlAaAhoQBaJkcAZCtkDoQAZA5lJ4MDWihkQWQsZC2EAVopZC5kDYQAWipkGmErZRZlFWUpZQlqLGUJai1mBWQvZDCEAVouZQSgL2UuoQEBAEcAZDFkD4QAZA9lJ4MDWjBHAGQyZBCEAGQQZQVqMYMDWjJ6DmUAoDNkM6EBWjRXAG4aBABlNWsKkAFypAEAAQABAGQ0WjRZAG4CWABkNWQRhABaNmQ2ZDeEAFo3ZDhkOYQAWjhkOmQ7hABaOWQ8ZD2EAFo6ZD5kP4QAWjtkAVMA
+ZABkAWwAWgBkAGQBbAFaAWQCZANsAm0DWgMBAGQEZwFaBEcAZAVkBIQAZARlBYMDWgZkAVMA
+ZABaAGQBZAJsAVoBZAFkAmwCWgJkAWQDbANtBFoEAQBkAWQEbAVtBloGAQBkAWQFbAdtB1oHAQBkAWQGbAhtCVoJAQBkAWQHbAptCloKAQBkCGQJbAttDFoMAQBkCGQKbAttDVoNAQBkCGQLbAttDloOAQBkCGQMbAttD1oPAQBkCGQNbAttEFoQAQBkCGQObAttEVoRAQBkCGQPbAttEloSAQBkCGQQbAttE1oTAQBkCGQRbAttFFoUAQBkCGQSbBVtFloWAQBkCGQTbBdtGFoYAQBkCGQUbBdtGVoZAQBkCGQVbBdtGloaAQBkCGQWbBdtG1obAQBkCGQXbBdtHFocAQBkCGQYbBdtHVodAQBkCGQZbBdtHloeAQBkCGQabB9tIFogAQBkCGQbbB9tIVohAQBkCGQcbB9tIloiAQBkCGQdbCNtJFokAQBkCGQebCNtJVolAQBkCGQfbCNtJlomAQBkCGQgbCNtJ1onAQBkCGQhbCNtKFooAQBkCGQibCNtKVopAQBkCGQjbCptK1orAQBkCGQkbCxtLVotAQBkCGQlbC5tL1ovAQBkCGQmbC5tMFowAQB6EGQBZCdsMW0yWjMBAFcAbiIEAGU0awqQAXLqAQABAAEAZAFkJ2w1bTJaMwEAWQBuAlgAehBkAWQobDZtN1o3AQBXAG4iBABlNGsKkAJyHgEAAQABAGQBZChsOG03WjcBAFkAbgJYAGRFZCxkLYQBWjlkRmQuZC+EAVo6ZEdkMGQxhAFaO0cAZDJkM4QAZDNlPIMDWj1HAGQ0ZDWEAGQ1ZTyDA1o+RwBkNmQ3hABkN2U3gwNaP2Q4ZDmEAFpARwBkOmQ7hABkO2U8gwNaQUcAZDxkPYQAZD1lQoMDWkNHAGQ+ZD+EAGQ/ZTyDA1pEZEBkQYQAWkVkSGRDZESEAVpGZAJTAA==
+ZABaAGQBZAJsAVoBZQJkA5wBZARkBYQEWgNkBmQHhABaBGQCUwA=
+ZABkAWwAWgFHAGQCZAOEAGQDZQFqAmoDgwNaBGQBUwA=
+ZABkAWwAWgBkAGQBbAFaAmQAZAFsA1oCZABkAWwEWgJkAGQBbAVaAkcAZAJkA4QAZANlAmoGageDA1oIZAFTAA==
+ZABkAWwAWgFkAGQBbAJaAUcAZAJkA4QAZANlAWoDagSDA1oFZAFTAA==
+ZABkAWwAWgBkAGQBbAFaAmQAZAFsA1oCZABkAWwEWgJkAGQBbAVaAmQAZAFsBloCZABkAmwHbQhaCAEAZABkA2wJbQpaCgEARwBkBGQFhABkBWUCagtqDIMDWg1kAVMA
+ZABaAGQBZAJsAW0CWgIBAGQBZANsA20EWgRtBVoFbQZaBgEAZAFkBGwHbQhaCG0JWgkBAGQBZAVsCm0LWgsBAGQGUwA=
+ZABkAWwAWgBkAGQBbAFaAWQAZAFsAloDZABkAWwEWgRkAGQCbAVtBloGAQBlAGoHoAhlAGoHoAllCqEBoQFkAxcAWgtkBGQFhABaDGQBUwA=
+ZABaAGQBZAJsAVoBZQGgAmUBagNkA6ECAQBkAWQCbARaBGQBZAJsBVoFZAFkBGwGbQdaBwEAZQRqCKAJZAFlBWoIoAplBWoIoAtlDKEBoQGhAgEAZAVaDWQBZAZsDm0PWg8BAGQHZAiEAFoQZAtkCWQKhAFaEWQCUwA=
+ZABaAGQBZAJsAVoBZQGgAmUBagNkA6ECAQBkAWQCbARaBGQBZAJsBVoFZAFkBGwGbQdaBwEAZQRqCKAJZAFlBWoIoAplBWoIoAtlDKEBoQGhAgEAZAVaDWQBZAZsDm0PWg8BAGQBZAdsEG0RWhEBAGQKZAhkCYQBWhJkAlMA
+ZABkAWwAWgBkAGQCbAFtAloCAQBkA2QEhABaA2QBUwA=
+ZABaAGQBZAJsAVoBZQGgAmUBagNkA6ECAQBkAWQCbARaBGQBZAJsBVoFZAFkAmwGWgZkAWQCbAdaB2QBZAJsCFoIZAFkAmwJWglkAWQCbApaCmQBZAJsC1oMZAFkBGwNbQ5aDwEAZAFkBWwQbRFaEW0SWhIBAGQBZAZsE20UWhRtFVoVbRZaFm0XWhcBAGQBZAJsGFoYZAFkAmwZWhlkB1oaRwBkCGQJhABkCYMCWhtkEWQKZAuEAVocZBJkDGQNhAFaHWQOZR1fHmQPZRxfHmQHZBBnAlofZAJTAA==
+ZABaAGQBZAJsAVoBZQJkA5wBZARkBYQEWgNlAmQDnAFkBmQHhARaBGQIZAmEAFoFZAJTAA==
+ZABkAWwAWgBkAGQCbAFtAloCAQBkAGQDbANtBFoEAQBkBGQFhABaBWQBUwA=
+ZABaAGQBZAJsAW0BWgEBAGQBZANsAm0DWgMBAGQBZARsBG0FWgUBAGQFZAZsBm0HWgcBAGUIZQhlCWUDZAecBGQIZAmEBFoKZApTAA==
+ZABkAWwAWgBkAGQBbAFaAWQAZAJsAm0DWgMBAGQAZANsBG0FWgUBAGQEZAWEAFoGZAFTAA==
+ZABkAWwAWgBkAGQBbAFaAWQAZAJsAm0DWgMBAGQAZANsBG0FWgUBAGQAZARsBm0HWgcBAGQFZAaEAFoIZAFTAA==
+ZABkAWwAWgBkAGQBbAFaAWQAZAJsAm0DWgMBAGQAZANsBG0FWgUBAGQAZARsBm0HWgcBAGQAZAVsCG0JWgkBAGQAZAZsCm0LWgsBAGQHZAiEAFoMZAFTAA==
+ZABkAWwAWgBkAGQBbAFaAWQAZAJsAm0DWgMBAGQAZANsBG0FWgUBAGQAZARsBm0HWgcBAGQAZAVsCG0JWgkBAGQAZAZsCm0LWgsBAGQAZAFsCFoIZAdkCIQAWgxkAVMA
+ZABaAGQBZAJsAW0CWgIBAGQBZANsA20EWgQBAGQBZARsBVoFZAFkBGwBWgFkAWQFbAZtB1oHAQBkBmQHhABaCGUJZAicAWQJZAqEBFoKZQlkCJwBZAtkDIQEWgtkDWQOhABaDGQEUwA=

static-data/connect-check.txt

@@ -1 +1 @@
-https://3g2upl4pq6kufc4m.onion/robots.txt,http://expyuzz4wqqyqhjn.onion/robots.txt,http://archivecaslytosk.onion/robots.txt,http://cockmailwwfvrtqj.onion/robots.txt
+https://3g2upl4pq6kufc4m.onion/robots.txt,http://expyuzz4wqqyqhjn.onion/robots.txt

static-data/default-plugins/encrypt/main.py

@@ -1,9 +1,21 @@
-'''
+"""Onionr - Private P2P Communication.
-    Onionr - Private P2P Communication
 
 This default plugin allows users to encrypt/decrypt messages without using blocks
-'''
+"""
-'''
+import locale
+locale.setlocale(locale.LC_ALL, '')
+import binascii
+
+# Imports some useful libraries
+import logger, config, threading, time, datetime, sys
+
+import ujson as json
+from nacl.exceptions import TypeError as NaclTypeError
+
+from onionrutils import stringvalidators, bytesconverter
+from onionrcrypto import encryption, keypair, signing, getourkeypair
+import onionrexceptions, onionrusers
+"""
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 the Free Software Foundation, either version 3 of the License, or
@@ -16,22 +28,7 @@
 
 You should have received a copy of the GNU General Public License
 along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
+"""
-
-# Imports some useful libraries
-import logger, config, threading, time, datetime, sys
-
-import ujson as json
-from nacl.exceptions import TypeError as NaclTypeError
-
-from onionrutils import stringvalidators, bytesconverter
-from onionrcrypto import encryption, keypair, signing, getourkeypair
-import onionrexceptions, onionrusers
-
-import locale
-locale.setlocale(locale.LC_ALL, '')
-
-import binascii
 
 plugin_name = 'encrypt'
 

static-data/default-plugins/torgossip/commandhandlers.py

@@ -1,13 +1,64 @@
+"""Onionr - Private P2P Communication.
+
+Handle commands for the torgossip server
+"""
+from onionrblocks import generators
+from onionrblocks.generators import anonvdf
 import blockio
 
+import onionrblocks
+from kasten import Kasten
+"""
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+def put_block(safe_db, block):
+    block_hash = block[:64]
+    data = block[64:]
+    try:
+        blockio.store_block(
+            Kasten(block_hash, data, onionrblocks.generators.AnonVDFGenerator),
+            safe_db)
+    except ValueError:
+        print("Block was seen before")
+        pass
+    except Exception as e:
+        print("Unknown error" + repr(e))
+        return b"0"
+    return b"1"
+
+
+
+def get_block(safe_db, block_hash) -> bytes:
+    # 4
+    try:
+        return safe_db.get(block_hash)
+    except KeyError:
+        return b"0"
+
 
 def list_blocks_by_type(safe_db, block_type) -> bytes:
+    # 3
+    block_type = block_type.decode('utf-8')
+    print('ty', block_type)
     try:
-        return safe_db.get(b'bl-' + block_type)
+        return safe_db.get('bl-' + block_type)
     except KeyError:
-        return b""
+        return b"0"
+
 
 def handle_check_block(safe_db, block_hash):
+    # 2
     if block_hash in blockio.list_all_blocks(safe_db):
         return int(1).to_bytes(1, 'little')
     else:

static-data/default-plugins/torgossip/commands.py

@@ -24,4 +24,6 @@ class GossipCommands(IntEnum):
     PING = 1,
     CHECK_HAS_BLOCK = 2,
     LIST_BLOCKS_BY_TYPE = 3,
-    EXIT = 4
+    GET_BLOCK = 4,
+    PUT_BLOCK = 5,
+    EXIT = 6

static-data/default-plugins/torgossip/main.py

@@ -25,9 +25,13 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 """
 plugin_name = 'torgossip'
 from server import start_server
+from runtest import torgossip_runtest
 
 def on_init(api, data=None):
     shared_state = data
-    print("starting gossip transport")
+
+    shared_state.get_by_string(
+        "OnionrRunTestManager").plugin_tests.append(torgossip_runtest)
+
     Thread(target=start_server, daemon=True, args=[shared_state]).start()
 

static-data/default-plugins/torgossip/runtest.py

@@ -0,0 +1,51 @@
+import socket
+import os
+from threading import local
+
+from utils import identifyhome
+from onionrblocks import blockcreator
+from blockio import subprocgenerate
+from onionrutils import localcommand
+import blockio
+
+def torgossip_runtest(test_manager):
+
+
+    s_file = identifyhome.identify_home() + "/torgossip.sock"
+    bl_test = blockcreator.create_anonvdf_block(b"test", "txt", 10)
+
+    #test_manager._too_many.get_by_string("PassToSafeDB").queue_then_store(b"test", "txt", 10)
+    bl = subprocgenerate.vdf_block(b"test", "txt", 100)
+    blockio.store_block(bl, test_manager._too_many.get_by_string("SafeDB"))
+
+    bl_new = blockcreator.create_anonvdf_block(b"test5", "txt", 10)
+
+
+    with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s:
+        s.connect(s_file)
+        s.sendall(b'1')
+        resp = s.recv(5)
+        assert resp == b"PONG"
+
+        s.sendall(b'3txx')
+        assert s.recv(64) == b"0"
+
+        s.sendall(b'3txt')
+        assert bl.id in s.recv(10000)
+
+        # test getting a block that doesn't exist
+        s.sendall(b'4' + os.urandom(64))
+        assert s.recv(64) == b"0"
+
+        # test getting a block that does exist
+        s.sendall(b'4' + bl.id)
+        assert s.recv(64) == bl.get_packed()
+
+        s.sendall(b'5' + bl_new.id + bl_new.get_packed())
+        assert s.recv(2) == b"1"
+
+        # test block was uploaded by getting it
+        s.sendall(b'4' + bl_new.id)
+        assert s.recv(64) == bl_new.get_packed()
+
+

static-data/default-plugins/torgossip/server.py

@@ -60,6 +60,16 @@ def start_server(shared_state):
                 conn.sendall(b'PONG')
             elif cmd == GossipCommands.EXIT:
                 do_close(conn, b'BYE')
+            elif cmd == GossipCommands.PUT_BLOCK:
+                conn.sendall(
+                    commandhandlers.put_block(
+                        shared_state.get_by_string('SafeDB'), data
+                    )
+                )
+            elif cmd == GossipCommands.GET_BLOCK:
+                conn.sendall(
+                    commandhandlers.get_block(
+                        shared_state.get_by_string('SafeDB'), data))
             elif cmd == GossipCommands.LIST_BLOCKS_BY_TYPE:
                 conn.sendall(
                     commandhandlers.list_blocks_by_type(

static-data/default-plugins/torgossip/server_test.py

@@ -10,7 +10,7 @@ def client(data):
     with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s:
         s.connect(f'{home}/torgossip.sock')
         s.sendall(data)
-        resp = s.recv(1024)
+        resp = s.recv(32)
         print("\n", resp)
 
 while True:

tests/runtime-result.txt

@@ -1 +0,0 @@
-1611612283

tests/test_blockcreatorqueue.py

@@ -23,8 +23,6 @@ setup_config()
 
 class TestBlockCreatorQueue(unittest.TestCase):
 
-    def test_in_queue(self):
-        
 
     def test_blockcreator_queue_1(self):
         received_callback = [False]