kev
/
Onionr
1
0
Fork 0
Code Issues 24 Pull Requests Projects Releases 2 Wiki Activity

fixed nasty bug where tor circuits was causing gevent to leak threads

This commit is contained in:
Kevin Froman 2020-03-20 03:50:48 -05:00
parent 022fbaa1af
commit e6181bdd1f
12 changed files with 47 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/communicatorutils/housekeeping.py
View File

@ -50,11 +50,12 @@ def clean_old_blocks(comm_inst):
oldest = blockmetadb.get_block_list()[0]
except IndexError:
break
blacklist.addToDB(oldest)
removeblock.remove_block(oldest)
onionrstorage.deleteBlock(oldest)
__remove_from_upload(comm_inst, oldest)
logger.info('Deleted block: %s' % (oldest,))
else:
blacklist.addToDB(oldest)
removeblock.remove_block(oldest)
onionrstorage.deleteBlock(oldest)
__remove_from_upload(comm_inst, oldest)
logger.info('Deleted block: %s' % (oldest,))
comm_inst.decrementThreadCount('clean_old_blocks')

9
src/communicatorutils/onionrcommunicatortimers.py
View File

@ -26,15 +26,16 @@ import onionrexceptions, logger
from typing import TYPE_CHECKING
from typing import Callable, NewType, Iterable
from psutil import Process
if TYPE_CHECKING:
from communicator import OnionrCommunicatorDaemon
CallFreqSeconds = NewType('CallFreqSeconds', int)
class OnionrCommunicatorTimers:
def __init__(self, daemon_inst: OnionrCommunicatorDaemon,
timer_function: Callable, frequency: CallFreqSeconds,
make_thread:bool=True, thread_amount:int=1, max_threads:int=5,
def __init__(self, daemon_inst: OnionrCommunicatorDaemon,
timer_function: Callable, frequency: CallFreqSeconds,
make_thread:bool=True, thread_amount:int=1, max_threads:int=5,
requires_peer:bool=False, my_args:Iterable=[]):
self.timer_function = timer_function
self.frequency = frequency
@ -70,7 +71,7 @@ class OnionrCommunicatorTimers:
logger.debug('%s is currently using the maximum number of threads, not starting another.' % self.timer_function.__name__)
else:
self.daemon_inst.threadCounts[self.timer_function.__name__] += 1
newThread = threading.Thread(target=self.timer_function, args=self.args, daemon=True,
newThread = threading.Thread(target=self.timer_function, args=self.args, daemon=True,
name=self.timer_function.__name__ + ' - ' + str(uuid.uuid4()))
newThread.start()
else:

8
src/httpapi/directconnections/__init__.py
View File

@ -49,21 +49,21 @@ class DirectConnectionManagement:
if pubkey in communicator.direct_connection_clients:
resp = communicator.direct_connection_clients[pubkey]
return Response(resp)
@direct_conn_management_bp.route('/dc-client/connect/<pubkey>')
def make_new_connection(pubkey):
communicator = _get_communicator(g)
resp = "pending"
if pubkey in communicator.shared_state.get(pool.ServicePool).bootstrap_pending:
return Response(resp)
if pubkey in communicator.direct_connection_clients:
resp = communicator.direct_connection_clients[pubkey]
else:
"""Spawn a thread that will create the client and eventually add it to the
communicator.active_services
communicator.active_services
"""
threading.Thread(target=onionrservices.OnionrServices().create_client,
threading.Thread(target=onionrservices.OnionrServices().create_client,
args=[pubkey, communicator], daemon=True).start()
return Response(resp)

1
src/onionrplugins/onionrevents.py
View File

@ -50,6 +50,7 @@ def event(event_name, data = {}, threaded = True):
'''
if threaded:
print('threaded event', event_name)
thread = Thread(target = __event_caller, args = (event_name, data))
thread.start()
return thread

1
src/onionrproofs/__init__.py
View File

@ -99,7 +99,6 @@ class POW:
self.mainHash = '0' * 64
self.puzzle = self.mainHash[0:min(self.difficulty, len(self.mainHash))]
for i in range(max(1, threadCount)):
t = threading.Thread(name = 'thread%s' % i, target = self.pow, args = (True,))
t.start()

8
src/onionrservices/bootstrapservice.py
View File

@ -41,7 +41,7 @@ def bootstrap_client_service(peer, comm_inst=None, bootstrap_timeout=300):
'''
if not stringvalidators.validate_pub_key(peer):
raise ValueError('Peer must be valid base32 ed25519 public key')
connection_pool = None
# here we use a lambda for the timeout thread to set to true
@ -60,7 +60,7 @@ def bootstrap_client_service(peer, comm_inst=None, bootstrap_timeout=300):
else:
comm_inst.service_greenlets.append(http_server)
connection_pool = comm_inst.shared_state.get(pool.ServicePool)
bootstrap_address = ''
shutdown = False
bs_id = str(uuid.uuid4())
@ -93,9 +93,9 @@ def bootstrap_client_service(peer, comm_inst=None, bootstrap_timeout=300):
controller.authenticate(config.get('tor.controlpassword'))
# Create the v3 onion service
response = controller.create_ephemeral_hidden_service({80: bootstrap_port}, key_type = 'NEW', key_content = 'ED25519-V3', await_publication = True)
onionrblocks.insert(response.service_id, header='con', sign=True, encryptType='asym',
onionrblocks.insert(response.service_id, header='con', sign=True, encryptType='asym',
asymPeer=peer, disableForward=True, expire=(epoch.get_epoch() + bootstrap_timeout))
threading.Thread(target=__bootstrap_timeout, args=[http_server, bootstrap_timeout, timed_out], daemon=True).start()
# Run the bootstrap server

25
src/onionrstatistics/serializeddata.py
View File

@ -1,8 +1,14 @@
"""
Onionr - Private P2P Communication
"""Onionr - Private P2P Communication.
This module serializes various data pieces for use in other modules, in particular the web api
Serialize various node information
"""
import json
from gevent import sleep
from psutil import Process
from coredb import blockmetadb
import communicator
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -18,11 +24,6 @@
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
import json # noqa
import time # noqa
from coredb import blockmetadb # noqa
import communicator # noqa
class SerializedData:
def __init__(self):
@ -34,19 +35,21 @@ class SerializedData:
etc
}
"""
def get_stats(self):
"""Return statistics about our node"""
stats = {}
proc = Process()
try:
self._too_many
except AttributeError:
time.sleep(1)
sleep(1)
comm_inst = self._too_many.get(communicator.OnionrCommunicatorDaemon, args=(self._too_many,))
connected = []
[connected.append(x) for x in comm_inst.onlinePeers if x not in connected]
[connected.append(x) for x in comm_inst.onlinePeers if x not in connected]
stats['uptime'] = comm_inst.getUptime()
stats['connectedNodes'] = '\n'.join(connected)
stats['blockCount'] = len(blockmetadb.get_block_list())
stats['blockQueueCount'] = len(comm_inst.blockQueue)
stats['threads'] = proc.num_threads()
return json.dumps(stats)

6
src/onionrstatistics/transports/tor/__init__.py
View File

@ -3,7 +3,6 @@
"""
import json
from gevent import sleep
from stem import CircStatus
@ -29,6 +28,7 @@ class TorStats:
def __init__(self):
self.circuits = {}
self.json_data = ""
self.controller = None
def get_json(self):
"""Refresh circuits then serialize them into form:
@ -36,6 +36,8 @@ class TorStats:
"nodes": list of tuples containing fingerprint and nickname strings"
"purpose": https://stem.torproject.org/api/control.html#stem.CircPurpose
"""
if self.controller is None:
self.controller = get_controller()
self.get_circuits()
json_serialized = {}
for circuit in self.circuits.keys():
@ -52,7 +54,7 @@ class TorStats:
def get_circuits(self):
"""Update the circuit dictionary"""
circuits = {}
for circ in list(sorted(get_controller().get_circuits())):
for circ in list(sorted(self.controller.get_circuits())):
if circ.status != CircStatus.BUILT:
continue
circuits[circ.id] = (circ.path, circ.purpose)

4
src/runtests/__init__.py
View File

@ -62,10 +62,10 @@ class OnionrRunTestManager:
for i in RUN_TESTS:
last = i
logger.info("[RUNTIME TEST] " + last.__name__ + " started",
terminal=True)
terminal=True, timestamp=True)
i(self)
logger.info("[RUNTIME TEST] " + last.__name__ + " passed",
terminal=True)
terminal=True, timestamp=True)
except (ValueError, AttributeError):
logger.error(last.__name__ + ' failed assertions', terminal=True)
except Exception as e:

5
src/runtests/clearnettor.py
View File

@ -2,13 +2,10 @@
Ensure that clearnet cannot be reached
"""
from threading import Thread
from onionrutils.basicrequests import do_get_request
from onionrutils import localcommand
import logger
import config
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -30,7 +27,7 @@ def test_clearnet_tor_request(testmanager):
Does not run if Tor is being reused
"""
config.reload()
leak_result = ""

6
src/runtests/lanservertest.py
View File

@ -10,11 +10,15 @@ def test_lan_server(testmanager):
try:
if requests.get(f"http://{best_ip}:{i}/ping").text == 'pong!':
bl = insert('test data')
if bl not in requests.get(f"http://{best_ip}:{i}/blist/0").text:
bl2 = insert('test data2')
l = requests.get(f"http://{best_ip}:{i}/blist/0").text
if bl not in l or bl2 not in l:
raise ValueError
if onionrblockapi.Block(bl).raw != requests.get(f"http://{best_ip}:{i}/get/{bl}").content:
raise ValueError
break
except requests.exceptions.ConnectionError:
pass
else:

2
tests/runtime-result.txt
View File

@ -1 +1 @@
1584597517
1584652661