work on lan

src/httpapi/sse/README.md

@@ -1,3 +1,6 @@
 # sse
 
-This folder contains a wrapper for handling server sent event loops
+This folder contains a wrapper for handling server sent event loops
+
+
+

src/lan/__init__.py

@@ -36,5 +36,4 @@ class LANManager:
     def start(self):
         Thread(target=learn_services, args=[self.too_many.get(Client)], daemon=True).start()
         Thread(target=advertise_service, daemon=True).start()
-        Thread(target=self.too_many.get(Client, (self.peers,)).start, daemon=True).start()
 

src/lan/client/__init__.py

@@ -4,9 +4,15 @@ LAN transport client thread
 """
 from typing import List
 
+import watchdog
+from requests.exceptions import ConnectionError
+
 from onionrcrypto.cryptoutils.randomshuffle import random_shuffle
 from utils.bettersleep import better_sleep
 from onionrutils.basicrequests import do_post_request, do_get_request
+from threading import Thread
+from onionrblocks import BlockList
+from onionrblocks.blockimporter import import_block_from_data
 """
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -29,17 +35,46 @@ class Client:
         self.lookup_time = {}
         self.poll_delay = 10
 
+
     def get_lookup_time(self, peer):
         try:
             return self.lookup_time[peer]
         except KeyError:
             return 0
 
-    def start(self):
+    def peer_work(self, peer):
+        port = 1024
+
+        self.peers.append(peer)
+        for port in range(port, 65535):
+            print(port)
+            try:
+                if do_get_request(f'http://{peer}:{port}/ping', proxyType='lan', ignoreAPI=True, connect_timeout=0.3) == 'onionr!':
+                    port = port
+                    print(f'{peer}:{port} found')
+                    break
+            except (AttributeError, ConnectionError):
+                pass
+        else:
+            self.peers.remove(peer)
+            return
+        self.peers.append(peer)
+
         while True:
-            self.peers = random_shuffle(self.peers)
+            block_list = self._too_many.get(BlockList).get()
+            last_time = self.get_lookup_time(peer)
+            new_blocks = set('\n'.join(do_get_request(f'http://{peer}:{port}/blist/{last_time}', proxyType='lan', ignoreAPI=True))) ^ set(block_list)
 
+            for bl in new_blocks:
+                import_block_from_data(
+                    do_get_request(
+                        f'http://{peer}:{port}/get/{bl}', proxyType='lan', ignoreAPI=True))
+            better_sleep(10)
+        self.peers.remove(peer)
 
-
+    def connect_peer(self, peer):
-            better_sleep(self.poll_delay)
+        if peer in self.peers:
+            return
+        print(f'connecting to {peer}')
+        Thread(target=self.peer_work, args=[peer], daemon=True).start()
 

src/lan/discover.py

@@ -9,7 +9,7 @@ from typing import List
 from ipaddress import ip_address
 from socket import SHUT_RDWR
 
-from .getip import lan_ips
+from .getip import lan_ips, best_ip
 from utils.bettersleep import better_sleep
 """
     This program is free software: you can redistribute it and/or modify
@@ -52,17 +52,17 @@ def learn_services(lan_client):
         if 'onionr' not in service_ips:
             continue
         service_ips = service_ips.replace('onionr-', '').split('-')
+        print(service_ips)
         port = 0
         for service in service_ips:
             try:
                 ip_address(service)
                 if not ip_address(service).is_private: raise ValueError
                 if service in lan_ips: raise ValueError
-                if service in lan_client.peers: raise ValueError
             except ValueError:
-                service_ips.remove(service)
+                pass
-        p = list(lan_client.peers)
+            else:
-        lan_client.peers = list(set(service_ips + p))
+                lan_client.connect_peer(service)
 
 
 def advertise_service(specific_ips=None):
@@ -71,10 +71,8 @@ def advertise_service(specific_ips=None):
     # for all packets sent, after three hops on the network the packet will not
     # be re-sent/broadcast (see https://www.tldp.org/HOWTO/Multicast-HOWTO-6.html)
     MULTICAST_TTL = 3
-    if specific_ips is None:
+
-        ips = '-'.join(lan_ips)
+    ips = best_ip
-    else:
-        ips = '-'.join(specific_ips)
 
     sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
     sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, MULTICAST_TTL)

src/lan/server/__init__.py

@@ -6,6 +6,7 @@ from gevent.pywsgi import WSGIServer
 from flask import Flask
 from flask import Response
 from flask import request
+from flask import abort
 
 from onionrblocks.onionrblockapi import Block
 from httpapi.fdsafehandler import FDSafeHandler
@@ -15,6 +16,7 @@ from coredb.blockmetadb import get_block_list
 from lan.getip import best_ip
 from onionrutils import stringvalidators
 from httpapi.miscpublicapi.upload import accept_upload
+import logger
 """
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -41,6 +43,13 @@ class LANServer:
             self.host = best_ip
         self.port = None
 
+        @app.before_request
+        def dns_rebinding_prevention():
+            if request.host != f'{self.host}:{self.port}':
+                logger.warn('Potential DNS rebinding attack on LAN server:')
+                logger.warn(f'Hostname {request.host} was used instead of {self.host}:{self.port}')
+                abort(403)
+
         @app.route('/blist/<time>')
         def get_block_list_for_lan(time):
             return Response('\n'.join(get_block_list(dateRec=time)))
@@ -54,7 +63,7 @@ class LANServer:
 
         @app.route("/ping")
         def ping():
-            return Response("pong!")
+            return Response("onionr!")
 
         @app.route('/upload', methods=['POST'])
         def upload_endpoint():
@@ -65,5 +74,5 @@ class LANServer:
                                  self.app, log=None,
                                  handler_class=FDSafeHandler)
         self.port = self.server.server_port
+        logger.info(f'Serving to LAN on {self.host}:{self.port}', terminal=True)
         self.server.serve_forever()
-

src/onionrblocks/__init__.py

@@ -1,5 +1,5 @@
 from . import insert
 from .insert import time_insert
-
+from .blocklist import BlockList
 insert = insert.insert_block
 time_insert = time_insert

src/onionrblocks/blocklist.py

@@ -0,0 +1,41 @@
+from threading import Thread
+
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler
+
+from utils.identifyhome import identify_home
+from coredb.dbfiles import block_meta_db
+from coredb.blockmetadb import get_block_list, get_blocks_by_type
+from onionrutils.epoch import get_epoch
+
+class BlockList:
+    def __init__(self, auto_refresh=True, block_type=''):
+        self.block_type = block_type
+        self.refresh_db()
+        self.check_time = get_epoch()
+
+        class Refresher(FileSystemEventHandler):
+            @staticmethod
+            def on_modified(event):
+                if event.src_path != block_meta_db:
+                    return
+                self.refresh_db()
+        if auto_refresh:
+            def auto_refresher():
+                observer = Observer()
+                observer.schedule(Refresher(), identify_home(), recursive=False)
+                observer.start()
+                while observer.is_alive():
+                    # call import func with timeout
+                    observer.join(120)
+            Thread(target=auto_refresher, daemon=True).start()
+
+    def get(self):
+        return self.block_list
+
+    def refresh_db(self):
+        self.check_time = get_epoch()
+        if not self.block_type:
+            self.block_list = get_block_list()
+        else:
+            self.block_list = get_blocks_by_type(self.block_type)

src/onionrutils/basicrequests.py

@@ -57,7 +57,7 @@ def do_post_request(url, data={}, port=0, proxyType='tor', max_size=10000, conte
     return retData
 
 
-def do_get_request(url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=False, max_size=5242880):
+def do_get_request(url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=False, max_size=5242880, connect_timeout=15):
     '''
     Do a get request through a local tor or i2p instance
     '''
@@ -72,7 +72,7 @@ def do_get_request(url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=
     elif proxyType == 'lan':
         address = urlparse(url).hostname
         if IPv4Address(address).is_private and not IPv4Address(address).is_loopback:
-            proxies = {}
+            proxies = None
         else:
             return
     else:
@@ -80,8 +80,9 @@ def do_get_request(url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=
     headers = {'User-Agent': 'PyOnionr', 'Connection':'close'}
     response_headers = dict()
     try:
-        proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
+        if not proxies is None:
-        r = streamedrequests.get(url, request_headers=headers, allow_redirects=False, proxy=proxies, connect_timeout=15, stream_timeout=120, max_size=max_size)
+            proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
+        r = streamedrequests.get(url, request_headers=headers, allow_redirects=False, proxy=proxies, connect_timeout=connect_timeout, stream_timeout=120, max_size=max_size)
         # Check server is using same API version as us
         if not ignoreAPI:
             try:

src/onionrutils/blockmetadata/fromdata.py

@@ -2,6 +2,7 @@
 
     Return a useful tuple of (metadata (header), meta, and data) by accepting raw block data
 """
+from json import JSONDecodeError
 import ujson as json
 
 from onionrutils import bytesconverter
@@ -37,7 +38,7 @@ def get_block_metadata_from_data(block_data):
 
     try:
         metadata = json.loads(bytesconverter.bytes_to_str(block_data[:block_data.find(b'\n')]))
-    except json.decoder.JSONDecodeError:
+    except JSONDecodeError:
         pass
     else:
         data = block_data[block_data.find(b'\n'):]

src/runtests/lanservertest.py

@@ -12,7 +12,7 @@ def test_lan_server(testmanager):
     start_time = get_epoch()
     for i in range(1024, 65536):
         try:
-            if requests.get(f"http://{best_ip}:{i}/ping").text == 'pong!':
+            if requests.get(f"http://{best_ip}:{i}/ping").text == 'onionr!':
                 bl = insert('test data')
                 sleep(10)
                 bl2 = insert('test data2')

static-data/default_config.json

@@ -7,16 +7,16 @@
     },
     "general": {
         "announce_node": true,
-        "dev_mode": false,
+        "dev_mode": true,
         "display_header": true,
         "ephemeral_tunnels": false,
         "hide_created_blocks": true,
-        "insert_deniable_blocks": true,
+        "insert_deniable_blocks": false,
         "max_block_age": 2678400,
-        "minimum_block_pow": 5,
+        "minimum_block_pow": 1,
-        "minimum_send_pow": 5,
+        "minimum_send_pow": 1,
         "public_key": "",
-        "random_bind_ip": true,
+        "random_bind_ip": false,
         "security_level": 0,
         "show_notifications": true,
         "store_plaintext_blocks": true,
@@ -30,12 +30,12 @@
         },
         "file": {
             "output": true,
-            "remove_on_exit": true
+            "remove_on_exit": false
         },
         "verbosity": "default"
     },
     "onboarding": {
-        "done": false
+        "done": true
     },
     "peers": {
         "max_connect": 1000,
@@ -68,7 +68,7 @@
     "transports": {
         "lan": true,
         "sneakernet": true,
-        "tor": true
+        "tor": false
     },
     "ui": {
         "theme": "dark"

tests/runtime-result.txt

@@ -1 +1 @@
-1585984468
+1586149428

tests/test_blocklist_class.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+import sys, os
+sys.path.append(".")
+sys.path.append("src/")
+import uuid
+TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
+print("Test directory:", TEST_DIR)
+os.environ["ONIONR_HOME"] = TEST_DIR
+import unittest
+
+from utils import identifyhome, createdirs, bettersleep
+from onionrsetup import setup_config
+from onionrblocks import BlockList, insert
+
+createdirs.create_dirs()
+setup_config()
+
+
+class TestBlockList(unittest.TestCase):
+    def test_block_list(self):
+        block_list = BlockList()
+        self.assertEqual(len(block_list.get()), 0)
+        bl = insert('test')
+        bettersleep.better_sleep(0.8)
+        self.assertIn(bl, block_list.get())
+        bl2 = insert('test2')
+        bettersleep.better_sleep(0.8)
+        self.assertIn(bl2, block_list.get())
+        self.assertIn(bl, block_list.get())
+
+unittest.main()