Work on torgossip client

Use stems HS validator instead of custom logic

src/bigbrother/ministry/ofexec.py

@@ -77,10 +77,10 @@ def block_exec(event, info):
     if code_b64 in whitelisted_source:
         return
     # uncomment when you want to build on the whitelist
-    else:
+    #else:
-        with open("../static-data/base64-code-whitelist.txt", "a") as f:
+    #    with open("../static-data/base64-code-whitelist.txt", "a") as f:
-            f.write(code_b64 + "\n")
+    #        f.write(code_b64 + "\n")
-        return
+    #    return
 
     for source in whitelisted_code:
         if info[0].co_filename.endswith(source):

src/onionrutils/stringvalidators.py

@@ -1,31 +1,37 @@
-'''
+"""
-    Onionr - Private P2P Communication
+Onionr - Private P2P Communication
 
-    validate various string data types
+validate various string data types
-'''
+"""
-'''
+import base64
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
 
-    This program is distributed in the hope that it will be useful,
+import unpaddedbase32
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
+import nacl.signing
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+import nacl.encoding
-    GNU General Public License for more details.
+from stem.util import tor_tools
 
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import base64, string
-import unpaddedbase32, nacl.signing, nacl.encoding
 from onionrutils import bytesconverter
+"""
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
+
 def validate_hash(data, length=64):
-    '''
+    """Validate if a string is a valid hash hex digest (does not compare, checks length and charset)
-        Validate if a string is a valid hash hex digest (does not compare, just checks length and charset)
 
     Length is only invalid if its *more* than the specified
-    '''
+    """
     retVal = True
     if data == False or data == True:
         return False
@@ -40,10 +46,9 @@ def validate_hash(data, length=64):
 
     return retVal
 
+
 def validate_pub_key(key):
-    '''
+    """Validate if a string is a valid base32 encoded Ed25519 key"""
-        Validate if a string is a valid base32 encoded Ed25519 key
-    '''
     if type(key) is type(None):
         return False
     # Accept keys that have no = padding
@@ -60,57 +65,15 @@ def validate_pub_key(key):
         retVal = True
     return retVal
 
-def validate_transport(id):
-    try:
-        idLength = len(id)
-        retVal = True
-        idNoDomain = ''
-        peerType = ''
-        # i2p b32 addresses are 60 characters long (including .b32.i2p)
-        if idLength == 60:
-            peerType = 'i2p'
-            if not id.endswith('.b32.i2p'):
-                retVal = False
-            else:
-                idNoDomain = id.split('.b32.i2p')[0]
-        # Onion v2's are 22 (including .onion), v3's are 62 with .onion
-        elif idLength == 22 or idLength == 62:
-            peerType = 'onion'
-            if not id.endswith('.onion'):
-                retVal = False
-            else:
-                idNoDomain = id.split('.onion')[0]
-        else:
-            retVal = False
-        if retVal:
-            if peerType == 'i2p':
-                try:
-                    id.split('.b32.i2p')[2]
-                except IndexError:
-                    pass
-                else:
-                    retVal = False
-            elif peerType == 'onion':
-                try:
-                    id.split('.onion')[2]
-                except IndexError:
-                    pass
-                else:
-                    retVal = False
-            if not idNoDomain.isalnum():
-                retVal = False
 
-            # Validate address is valid base32 (when capitalized and minus extension); v2/v3 onions and .b32.i2p use base32
+def validate_transport(id: str):
-            for x in idNoDomain.upper():
+    id = id.replace('.onion', '')
-                if x not in string.ascii_uppercase and x not in '234567':
+    return tor_tools.is_valid_hidden_service_address(
-                    retVal = False
+        id, version=3) and id.endswith('d')
 
-        return retVal
-    except Exception as e:
-        return False
 
 def is_integer_string(data):
-    '''Check if a string is a valid base10 integer (also returns true if already an int)'''
+    """Check if a string is a valid base10 integer (also returns true if already an int)"""
     try:
         int(data)
     except (ValueError, TypeError) as e:

src/runtests/__init__.py

@@ -18,6 +18,7 @@ from .housekeeping import test_inserted_housekeeping
 from .lanservertest import test_lan_server
 from .sneakernettest import test_sneakernet_import
 from .dnsrebindingtest import test_dns_rebinding
+from .serviceonlinetest import test_service_online
 """
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -43,8 +44,9 @@ RUN_TESTS = [uicheck.check_ui,
              test_clearnet_tor_request,
              test_inserted_housekeeping,
              test_lan_server,
-             sneakernettest.test_sneakernet_import,
+             test_sneakernet_import,
-             test_dns_rebinding
+             test_dns_rebinding,
+             test_service_online
              ]
 
 SUCCESS_FILE = os.path.dirname(os.path.realpath(__file__)) + '/../../tests/runtime-result.txt'

src/runtests/sneakernettest.py

@@ -15,6 +15,8 @@ from gevent import sleep
 def test_sneakernet_import(test_manager):
     if not config.get('transports.lan', False):
         return
+    if config.get('runtests.skip_slow', False):
+        return
     in_db = lambda b: b in get_block_list()
     bl = insert(os.urandom(10))
     assert in_db(bl)

static-data/default-plugins/torgossip/client.py

@@ -4,6 +4,15 @@ Torgossip client
 
 Create streams to random peers
 """
+from os import path
+from typing import TYPE_CHECKING
+from random import SystemRandom
+
+from netcontroller.torcontrol.onionserviceonline import service_online_recently
+from netcontroller.torcontrol import torcontroller
+
+if TYPE_CHECKING:
+    from .peerdb import TorGossipPeers
 """
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -20,7 +29,35 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 """
 
 
+def _add_bootstrap_peers(peer_db: 'TorGossipPeers'):
+    bootstap_peers = path.dirname(path.realpath(__file__)) + "/bootstrap.txt"
+    with open(bootstap_peers, 'r') as bs_peers:
+        peers = bs_peers.split(',')
+        for peer in peers:
+            if peer:
+                peer_db.add_peer(peer)
+
+
+def _client_pool(shared_state, controller):
+    peer_db: 'TorGossipPeers' = shared_state.get_by_string('TorGossipPeers')
+    socket_pool = {}
+
+    peers = peer_db.get_highest_score_peers(20)
+    SystemRandom().shuffle(peers)
+
+    for peer in peers:
+        if not service_online_recently(controller, peer):
+            continue
+
 def client_pool(shared_state):
-    return
+    controller = torcontroller.get_controller()
+    # Pass the stem Controller and then close it even if an exception raises
+    try:
+        _client_pool(shared_state, controller)
+    except Exception:
+        raise
+    finally:
+        # Calls before the above raise no matter what
+        controller.close()
 
 

static-data/default-plugins/torgossip/peerdb.py

@@ -5,6 +5,7 @@ Torgossip peer safedb interface
 from base64 import b32decode
 from struct import unpack, pack
 from time import time
+from typing import List
 
 from utils.identifyhome import identify_home
 import safedb
@@ -54,7 +55,7 @@ class TorGossipPeers:  # name it this way to avoid collisions in SharedState
 
         self.db.put(peer, pack(self.PACK_FORMAT, score, seen))
 
-    def get_highest_score_peers(self, max):
+    def get_highest_score_peers(self, max) -> List:
         assert max >= 1
         peer = self.db.db_conn.firstkey()
         if peer == b'enc':

static-data/default-plugins/torgossip/runtest.py

@@ -112,5 +112,3 @@ def torgossip_runtest(test_manager):
 
         s.sendall(b'9')
         assert s.recv(64) == b"BYE"
-
-

tests/test_database_actions.py

@@ -17,7 +17,7 @@ from coredb import keydb
 class OnionrTests(unittest.TestCase):
 
     def test_address_add(self):
-        testAddresses = ['facebookcorewwwi.onion', '56kmnycrvepfarolhnx6t2dvmldfeyg7jdymwgjb7jjzg47u2lqw2sad.onion', '5bvb5ncnfr4dlsfriwczpzcvo65kn7fnnlnt2ln7qvhzna2xaldq.b32.i2p']
+        testAddresses = ['56kmnycrvepfarolhnx6t2dvmldfeyg7jdymwgjb7jjzg47u2lqw2sad.onion', 'ao34zusas5oocjllkh6uounorhtujyep4ffwz4k4r7qkxie5otdiwqad.onion']
         for address in testAddresses:
             keydb.addkeys.add_address(address)
         dbAddresses = keydb.listkeys.list_adders()
@@ -35,7 +35,7 @@ class OnionrTests(unittest.TestCase):
             self.assertNotIn(address, dbAddresses)
 
     def test_address_info(self):
-        adder = 'nytimes3xbfgragh.onion'
+        adder = 'ao34zusas5oocjllkh6uounorhtujyep4ffwz4k4r7qkxie5otdiwqad.onion'
         keydb.addkeys.add_address(adder)
         self.assertNotEqual(keydb.transportinfo.get_address_info(adder, 'success'), 1000)
         keydb.transportinfo.set_address_info(adder, 'success', 1000)

tests/test_stringvalidations.py

@@ -12,10 +12,9 @@ class OnionrValidations(unittest.TestCase):
 
     def test_peer_validator(self):
         # Test hidden service domain validities
-        valid = ['facebookcorewwwi.onion', 'vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd.onion', 
+        valid = ['ao34zusas5oocjllkh6uounorhtujyep4ffwz4k4r7qkxie5otdiwqad.onion', 'vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd.onion']
-        '5bvb5ncnfr4dlsfriwczpzcvo65kn7fnnlnt2ln7qvhzna2xaldq.b32.i2p']
 
-        invalid = [None, 'dsfewjirji0ejipdfs', '', '    ', '\n', '\r\n', 'f$ce%^okc+rewwwi.onion', 'facebookc0rewwi.onion']
+        invalid = [None, 'dsfewjirji0ejipdfs', '', '    ', '\n', '\r\n', 'f$ce%^okc+rewwwi.onion', 'facebookc0rewwi.onion', 'facebookcorewwwi.onion']
 
         for x in valid:
             print('testing', x)