kev/Onionr
1
0
Fork 0
Code Issues 24 Pull Requests Projects Releases 2 Wiki Activity

added flood fill work, misc bug fixes and removing communicator timers

Browse Source
This commit is contained in:
Kevin Froman 2020-11-13 08:17:48 +00:00
parent 04105a2b70
commit bbd76da333
15 changed files with 98 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/apiservers/private/register_private_blueprints.py
View File

@ -5,7 +5,7 @@ This file registers blueprints for the private api server
from threading import Thread
from gevent import sleep
from httpapi import security, friendsapi, profilesapi, configapi, insertblock
from httpapi import security, friendsapi, configapi, insertblock
from httpapi import miscclientapi, onionrsitesapi, apiutils
from httpapi import directconnections
from httpapi import themeapi
@ -33,7 +33,6 @@ def register_private_blueprints(private_api, app):
app.register_blueprint(security.client.ClientAPISecurity(
private_api).client_api_security_bp)
app.register_blueprint(friendsapi.friends)
app.register_blueprint(profilesapi.profile_BP)
app.register_blueprint(configapi.config_BP)
app.register_blueprint(insertblock.ib)
app.register_blueprint(miscclientapi.getblocks.client_get_blocks)

15
src/communicator/__init__.py
View File

@ -86,12 +86,10 @@ class OnionrCommunicatorDaemon:
# extends our upload list and saves our list when Onionr exits
uploadqueue.UploadQueue(self)
# Timers to periodically lookup new blocks and download them
lookup_blocks_timer = OnionrCommunicatorTimers(
self,
add_onionr_thread(
lookupblocks.lookup_blocks_from_communicator,
config.get('timers.lookupBlocks', 25),
my_args=[self], requires_peer=True, max_threads=1)
[self.shared_state], 25, 3)
"""The block download timer is accessed by the block lookup function
to trigger faster download starts"""
@ -99,11 +97,7 @@ class OnionrCommunicatorDaemon:
self, self.getBlocks, config.get('timers.getBlocks', 10),
requires_peer=True, max_threads=5)
# Timer to reset the longest offline peer
# so contact can be attempted again
OnionrCommunicatorTimers(
self, onlinepeers.clear_offline_peer, 58, my_args=[self],
max_threads=1)
add_onionr_thread(onlinepeers.clear_offline_peer, [self.kv], 58)
# Timer to cleanup old blocks
blockCleanupTimer = OnionrCommunicatorTimers(
@ -180,7 +174,6 @@ class OnionrCommunicatorDaemon:
# Adjust initial timer triggers
cleanupTimer.count = (cleanupTimer.frequency - 60)
blockCleanupTimer.count = (blockCleanupTimer.frequency - 2)
lookup_blocks_timer = (lookup_blocks_timer.frequency - 2)
shared_state.add(self)

5
src/communicator/onlinepeers/clearofflinepeer.py
View File

@ -23,9 +23,8 @@ if TYPE_CHECKING:
"""
def clear_offline_peer(comm_inst: 'OnionrCommunicatorDaemon'):
def clear_offline_peer(kv: 'DeadSimpleKV'):
"""Remove the longest offline peer to retry later."""
kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string("DeadSimpleKV")
try:
removed = kv.get('offlinePeers').pop(0)
except IndexError:
@ -33,4 +32,4 @@ def clear_offline_peer(comm_inst: 'OnionrCommunicatorDaemon'):
else:
logger.debug('Removed ' + removed +
' from offline list, will try them again.')
comm_inst.decrementThreadCount('clear_offline_peer')

9
src/communicatorutils/lookupblocks.py
View File

@ -39,7 +39,7 @@ blacklist = onionrblacklist.OnionrBlackList()
storage_counter = StorageCounter()
def lookup_blocks_from_communicator(comm_inst):
def lookup_blocks_from_communicator(shared_state: 'TooMany'):
logger.info('Looking up new blocks')
tryAmount = 2
newBlocks = ''
@ -50,7 +50,7 @@ def lookup_blocks_from_communicator(comm_inst):
maxBacklog = 1560
lastLookupTime = 0 # Last time we looked up a particular peer's list
new_block_count = 0
kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string("DeadSimpleKV")
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
for i in range(tryAmount):
# Defined here to reset it each time, time offset is added later
listLookupCommand = 'getblocklist'
@ -87,7 +87,7 @@ def lookup_blocks_from_communicator(comm_inst):
listLookupCommand += '?date=%s' % (lastLookupTime,)
try:
newBlocks = peeraction.peer_action(
comm_inst.shared_state,
shared_state,
peer, listLookupCommand) # get list of new block hashes
except Exception as error:
logger.warn(
@ -124,6 +124,3 @@ def lookup_blocks_from_communicator(comm_inst):
logger.info(
f'Discovered {new_block_count} new block{block_string}',
terminal=True)
comm_inst.download_blocks_timer.count = \
int(comm_inst.download_blocks_timer.frequency * 0.99)
comm_inst.decrementThreadCount('lookup_blocks_from_communicator')

21
src/httpapi/miscclientapi/endpoints.py
View File

@ -8,6 +8,7 @@ import platform
from flask import Response, Blueprint, request, send_from_directory, abort
from flask import g
from gevent import sleep
import unpaddedbase32
from httpapi import apiutils
@ -138,10 +139,6 @@ class PrivateEndpoints:
return Response(
bytesconverter.bytes_to_str(mnemonickeys.get_base32(words)))
@private_endpoints_bp.route('/gettorsocks')
def get_tor_socks():
return Response(str(g.too_many.get(NetController).socksPort))
@private_endpoints_bp.route('/setonboarding', methods=['POST'])
def set_onboarding():
return Response(
@ -151,11 +148,25 @@ class PrivateEndpoints:
def get_os_system():
return Response(platform.system().lower())
@private_endpoints_bp.route('/gettorsocks')
def get_tor_socks():
while True:
try:
return Response(
str(
g.too_many.get_by_string(
'NetController').socksPort))
except KeyError:
sleep(0.1)
@private_endpoints_bp.route('/torready')
def is_tor_ready():
"""If Tor is starting up, the web UI is not ready to be used."""
try:
return Response(
str(g.too_many.get(NetController).readyState).lower())
str(g.too_many.get_by_string('NetController').readyState).lower())
except KeyError:
return Response("false")
@private_endpoints_bp.route('/gettoraddress')
def get_tor_address():

27
src/httpapi/profilesapi/__init__.py
View File

@ -1,27 +0,0 @@
'''
Onionr - Private P2P Communication
This file creates http endpoints for user profile pages
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
from flask import Blueprint, Response, request, abort
from . import profiles
profile_BP = Blueprint('profile_BP', __name__)
@profile_BP.route('/profile/get/<pubkey>', endpoint='profiles')
def get_profile_page(pubkey):
return Response(pubkey)

2
src/httpapi/profilesapi/profiles.py
View File

@ -1,2 +0,0 @@
def get_latest_user_profile(pubkey):
return ''

4
src/lan/client/__init__.py
View File

@ -10,6 +10,7 @@ from onionrtypes import LANIP
import logger
from coredb.blockmetadb import get_block_list
from onionrblocks.blockimporter import import_block_from_data
import onionrexceptions
from ..server import ports
from threading import Thread
@ -37,9 +38,12 @@ def _lan_work(peer: LANIP):
blocks = requests.get(url + 'blist/0').text.splitlines()
for block in blocks:
if block not in our_blocks:
try:
import_block_from_data(
requests.get(
url + f'get/{block}', stream=True).raw.read(6000000))
except onionrexceptions.InvalidMetadata:
logger.warn(f"Could not get {block} from lan peer")
for port in ports:
try:

8
src/onionrthreads/__init__.py
View File

@ -1,17 +1,25 @@
from typing import Callable
from typing import Iterable
import traceback
from threading import Thread
from time import sleep
import logger
def _onionr_thread(func: Callable, args: Iterable,
sleep_secs: int, initial_sleep):
if initial_sleep:
sleep(initial_sleep)
while True:
try:
func(*args)
except Exception as _: # noqa
logger.warn(
"Onionr thread exception \n" + traceback.format_exc(),
terminal=True)
sleep(sleep_secs)

4
src/runtests/clearnettor.py
View File

@ -44,14 +44,14 @@ def test_clearnet_tor_request(testmanager):
try:
leak_result: str = do_get_request(
'https://onionr.net/404',
'https://example.com/notvalidpage',
port=socks_port, ignoreAPI=True).lower()
except AttributeError:
leak_result = ""
except Exception as e:
logger.warn(str(e))
try:
if 'not found' in leak_result:
if 'example' in leak_result:
logger.error('Tor was able to request a clearnet site')
raise ValueError('Tor was able to request a clearnet site')
except TypeError:

4
src/sneakernet/__init__.py
View File

@ -45,7 +45,9 @@ class _Importer(FileSystemEventHandler):
os.remove(event.src_path)
try:
import_block_from_data(block_data)
except onionrexceptions.DataExists:
except(
onionrexceptions.DataExists,
onionrexceptions.BlockMetaEntryExists) as _:
return
if block_data_location in event.src_path:
try:

10
src/streamfill/pool.py Normal file
View File

@ -0,0 +1,10 @@
from typing import List
from onionrutils.localcommand import local_command
from .neighbors import identify_neighbors
def stream_pool():
peers = lioc

38
src/streamfill/tunnel/__init__.py Normal file
View File

@ -0,0 +1,38 @@
from secrets import token_bytes
from typing import TYPE_CHECKING
import socket
if TYPE_CHECKING:
from stem.control import Controller
from onionrtypes import OnionAddressString
import yam
def peer_tunnel(tor_controller: Controller, peer):
socks_port = tor_controller.get_conf('SocksPort')
class Connected:
connected = False
send_buffer = []
rec_buffer = []
rec_address = None
yam.client(1, peer, socks_port, send_buffer, rec_buffer, Connected)
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
ip = '127.0.0.1'
s.bind((ip, 0))
s.listen(1)
port = s.getsockname()[1]
serv = tor_controller.create_ephemeral_hidden_service(
{1337: '127.0.0.1:' + str(port)},
key_content='ED25519-V3',
await_publication=True,
)
rec_address = serv.service_id
conn, addr = s.accept()
yam.server(1, tor_controller, conn, send_buffer, rec_buffer, Connected)

16
tests/test_random_bind_ip_setting.py
View File

@ -1,16 +0,0 @@
import sys, os
sys.path.append(".")
sys.path.append("src/")
import unittest, uuid
TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
print("Test directory:", TEST_DIR)
os.environ["ONIONR_HOME"] = TEST_DIR
from onionrsetup import setup_config
setup_config()
import config
class TestRandomBindIP(unittest.TestCase):
def test_random_bind_ip_default_setting(self):
self.assertTrue(config.get('general.random_bind_ip'))
unittest.main()