kev/Onionr
1
0
Fork 0
Code Issues 24 Pull Requests Projects Releases 2 Wiki Activity

work on lan

Browse Source
This commit is contained in:
Kevin Froman 2020-04-06 08:51:20 -05:00
parent a5983d32a4
commit a52465a54f
13 changed files with 151 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/httpapi/sse/README.md
View File

@ -1,3 +1,6 @@
# sse
This folder contains a wrapper for handling server sent event loops

1
src/lan/__init__.py
View File

@ -36,5 +36,4 @@ class LANManager:
def start(self):
Thread(target=learn_services, args=[self.too_many.get(Client)], daemon=True).start()
Thread(target=advertise_service, daemon=True).start()
Thread(target=self.too_many.get(Client, (self.peers,)).start, daemon=True).start()

43
src/lan/client/__init__.py
View File

@ -4,9 +4,15 @@ LAN transport client thread
"""
from typing import List
import watchdog
from requests.exceptions import ConnectionError
from onionrcrypto.cryptoutils.randomshuffle import random_shuffle
from utils.bettersleep import better_sleep
from onionrutils.basicrequests import do_post_request, do_get_request
from threading import Thread
from onionrblocks import BlockList
from onionrblocks.blockimporter import import_block_from_data
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -29,17 +35,46 @@ class Client:
self.lookup_time = {}
self.poll_delay = 10
def get_lookup_time(self, peer):
try:
return self.lookup_time[peer]
except KeyError:
return 0
def start(self):
def peer_work(self, peer):
port = 1024
self.peers.append(peer)
for port in range(port, 65535):
print(port)
try:
if do_get_request(f'http://{peer}:{port}/ping', proxyType='lan', ignoreAPI=True, connect_timeout=0.3) == 'onionr!':
port = port
print(f'{peer}:{port} found')
break
except (AttributeError, ConnectionError):
pass
else:
self.peers.remove(peer)
return
self.peers.append(peer)
while True:
self.peers = random_shuffle(self.peers)
block_list = self._too_many.get(BlockList).get()
last_time = self.get_lookup_time(peer)
new_blocks = set('\n'.join(do_get_request(f'http://{peer}:{port}/blist/{last_time}', proxyType='lan', ignoreAPI=True))) ^ set(block_list)
for bl in new_blocks:
import_block_from_data(
do_get_request(
f'http://{peer}:{port}/get/{bl}', proxyType='lan', ignoreAPI=True))
better_sleep(10)
self.peers.remove(peer)
better_sleep(self.poll_delay)
def connect_peer(self, peer):
if peer in self.peers:
return
print(f'connecting to {peer}')
Thread(target=self.peer_work, args=[peer], daemon=True).start()

16
src/lan/discover.py
View File

@ -9,7 +9,7 @@ from typing import List
from ipaddress import ip_address
from socket import SHUT_RDWR
from .getip import lan_ips
from .getip import lan_ips, best_ip
from utils.bettersleep import better_sleep
"""
This program is free software: you can redistribute it and/or modify
@ -52,17 +52,17 @@ def learn_services(lan_client):
if 'onionr' not in service_ips:
continue
service_ips = service_ips.replace('onionr-', '').split('-')
print(service_ips)
port = 0
for service in service_ips:
try:
ip_address(service)
if not ip_address(service).is_private: raise ValueError
if service in lan_ips: raise ValueError
if service in lan_client.peers: raise ValueError
except ValueError:
service_ips.remove(service)
p = list(lan_client.peers)
lan_client.peers = list(set(service_ips + p))
pass
else:
lan_client.connect_peer(service)
def advertise_service(specific_ips=None):
@ -71,10 +71,8 @@ def advertise_service(specific_ips=None):
# for all packets sent, after three hops on the network the packet will not
# be re-sent/broadcast (see https://www.tldp.org/HOWTO/Multicast-HOWTO-6.html)
MULTICAST_TTL = 3
if specific_ips is None:
ips = '-'.join(lan_ips)
else:
ips = '-'.join(specific_ips)
ips = best_ip
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, MULTICAST_TTL)

13
src/lan/server/__init__.py
View File

@ -6,6 +6,7 @@ from gevent.pywsgi import WSGIServer
from flask import Flask
from flask import Response
from flask import request
from flask import abort
from onionrblocks.onionrblockapi import Block
from httpapi.fdsafehandler import FDSafeHandler
@ -15,6 +16,7 @@ from coredb.blockmetadb import get_block_list
from lan.getip import best_ip
from onionrutils import stringvalidators
from httpapi.miscpublicapi.upload import accept_upload
import logger
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -41,6 +43,13 @@ class LANServer:
self.host = best_ip
self.port = None
@app.before_request
def dns_rebinding_prevention():
if request.host != f'{self.host}:{self.port}':
logger.warn('Potential DNS rebinding attack on LAN server:')
logger.warn(f'Hostname {request.host} was used instead of {self.host}:{self.port}')
abort(403)
@app.route('/blist/<time>')
def get_block_list_for_lan(time):
return Response('\n'.join(get_block_list(dateRec=time)))
@ -54,7 +63,7 @@ class LANServer:
@app.route("/ping")
def ping():
return Response("pong!")
return Response("onionr!")
@app.route('/upload', methods=['POST'])
def upload_endpoint():
@ -65,5 +74,5 @@ class LANServer:
self.app, log=None,
handler_class=FDSafeHandler)
self.port = self.server.server_port
logger.info(f'Serving to LAN on {self.host}:{self.port}', terminal=True)
self.server.serve_forever()

2
src/onionrblocks/__init__.py
View File

@ -1,5 +1,5 @@
from . import insert
from .insert import time_insert
from .blocklist import BlockList
insert = insert.insert_block
time_insert = time_insert

41
src/onionrblocks/blocklist.py Normal file
View File

@ -0,0 +1,41 @@
from threading import Thread
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
from utils.identifyhome import identify_home
from coredb.dbfiles import block_meta_db
from coredb.blockmetadb import get_block_list, get_blocks_by_type
from onionrutils.epoch import get_epoch
class BlockList:
def __init__(self, auto_refresh=True, block_type=''):
self.block_type = block_type
self.refresh_db()
self.check_time = get_epoch()
class Refresher(FileSystemEventHandler):
@staticmethod
def on_modified(event):
if event.src_path != block_meta_db:
return
self.refresh_db()
if auto_refresh:
def auto_refresher():
observer = Observer()
observer.schedule(Refresher(), identify_home(), recursive=False)
observer.start()
while observer.is_alive():
# call import func with timeout
observer.join(120)
Thread(target=auto_refresher, daemon=True).start()
def get(self):
return self.block_list
def refresh_db(self):
self.check_time = get_epoch()
if not self.block_type:
self.block_list = get_block_list()
else:
self.block_list = get_blocks_by_type(self.block_type)

9
src/onionrutils/basicrequests.py
View File

@ -57,7 +57,7 @@ def do_post_request(url, data={}, port=0, proxyType='tor', max_size=10000, conte
return retData
def do_get_request(url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=False, max_size=5242880):
def do_get_request(url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=False, max_size=5242880, connect_timeout=15):
'''
Do a get request through a local tor or i2p instance
'''
@ -72,7 +72,7 @@ def do_get_request(url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=
elif proxyType == 'lan':
address = urlparse(url).hostname
if IPv4Address(address).is_private and not IPv4Address(address).is_loopback:
proxies = {}
proxies = None
else:
return
else:
@ -80,8 +80,9 @@ def do_get_request(url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=
headers = {'User-Agent': 'PyOnionr', 'Connection':'close'}
response_headers = dict()
try:
proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
r = streamedrequests.get(url, request_headers=headers, allow_redirects=False, proxy=proxies, connect_timeout=15, stream_timeout=120, max_size=max_size)
if not proxies is None:
proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
r = streamedrequests.get(url, request_headers=headers, allow_redirects=False, proxy=proxies, connect_timeout=connect_timeout, stream_timeout=120, max_size=max_size)
# Check server is using same API version as us
if not ignoreAPI:
try:

3
src/onionrutils/blockmetadata/fromdata.py
View File

@ -2,6 +2,7 @@
Return a useful tuple of (metadata (header), meta, and data) by accepting raw block data
"""
from json import JSONDecodeError
import ujson as json
from onionrutils import bytesconverter
@ -37,7 +38,7 @@ def get_block_metadata_from_data(block_data):
try:
metadata = json.loads(bytesconverter.bytes_to_str(block_data[:block_data.find(b'\n')]))
except json.decoder.JSONDecodeError:
except JSONDecodeError:
pass
else:
data = block_data[block_data.find(b'\n'):]

2
src/runtests/lanservertest.py
View File

@ -12,7 +12,7 @@ def test_lan_server(testmanager):
start_time = get_epoch()
for i in range(1024, 65536):
try:
if requests.get(f"http://{best_ip}:{i}/ping").text == 'pong!':
if requests.get(f"http://{best_ip}:{i}/ping").text == 'onionr!':
bl = insert('test data')
sleep(10)
bl2 = insert('test data2')

16
static-data/default_config.json
View File

@ -7,16 +7,16 @@
},
"general": {
"announce_node": true,
"dev_mode": false,
"dev_mode": true,
"display_header": true,
"ephemeral_tunnels": false,
"hide_created_blocks": true,
"insert_deniable_blocks": true,
"insert_deniable_blocks": false,
"max_block_age": 2678400,
"minimum_block_pow": 5,
"minimum_send_pow": 5,
"minimum_block_pow": 1,
"minimum_send_pow": 1,
"public_key": "",
"random_bind_ip": true,
"random_bind_ip": false,
"security_level": 0,
"show_notifications": true,
"store_plaintext_blocks": true,
@ -30,12 +30,12 @@
},
"file": {
"output": true,
"remove_on_exit": true
"remove_on_exit": false
},
"verbosity": "default"
},
"onboarding": {
"done": false
"done": true
},
"peers": {
"max_connect": 1000,
@ -68,7 +68,7 @@
"transports": {
"lan": true,
"sneakernet": true,
"tor": true
"tor": false
},
"ui": {
"theme": "dark"

2
tests/runtime-result.txt
View File

@ -1 +1 @@
1585984468
1586149428

31
tests/test_blocklist_class.py Normal file
View File

@ -0,0 +1,31 @@
#!/usr/bin/env python3
import sys, os
sys.path.append(".")
sys.path.append("src/")
import uuid
TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
print("Test directory:", TEST_DIR)
os.environ["ONIONR_HOME"] = TEST_DIR
import unittest
from utils import identifyhome, createdirs, bettersleep
from onionrsetup import setup_config
from onionrblocks import BlockList, insert
createdirs.create_dirs()
setup_config()
class TestBlockList(unittest.TestCase):
def test_block_list(self):
block_list = BlockList()
self.assertEqual(len(block_list.get()), 0)
bl = insert('test')
bettersleep.better_sleep(0.8)
self.assertIn(bl, block_list.get())
bl2 = insert('test2')
bettersleep.better_sleep(0.8)
self.assertIn(bl2, block_list.get())
self.assertIn(bl, block_list.get())
unittest.main()