fixed nasty bug where tor circuits was causing gevent to leak threads

src/communicatorutils/housekeeping.py

@@ -50,11 +50,12 @@ def clean_old_blocks(comm_inst):
             oldest = blockmetadb.get_block_list()[0]
         except IndexError:
             break
-        blacklist.addToDB(oldest)
-        removeblock.remove_block(oldest)
-        onionrstorage.deleteBlock(oldest)
-        __remove_from_upload(comm_inst, oldest)
-        logger.info('Deleted block: %s' % (oldest,))
+        else:
+            blacklist.addToDB(oldest)
+            removeblock.remove_block(oldest)
+            onionrstorage.deleteBlock(oldest)
+            __remove_from_upload(comm_inst, oldest)
+            logger.info('Deleted block: %s' % (oldest,))
 
     comm_inst.decrementThreadCount('clean_old_blocks')
 

src/communicatorutils/onionrcommunicatortimers.py

@@ -26,6 +26,7 @@ import onionrexceptions, logger
 
 from typing import TYPE_CHECKING
 from typing import Callable, NewType, Iterable
+from psutil import Process
 if TYPE_CHECKING:
     from communicator import OnionrCommunicatorDaemon
 

src/onionrplugins/onionrevents.py

@@ -50,6 +50,7 @@ def event(event_name, data = {}, threaded = True):
     '''
 
     if threaded:
+        print('threaded event', event_name)
         thread = Thread(target = __event_caller, args = (event_name, data))
         thread.start()
         return thread

src/onionrproofs/__init__.py

@@ -99,7 +99,6 @@ class POW:
 
         self.mainHash = '0' * 64
         self.puzzle = self.mainHash[0:min(self.difficulty, len(self.mainHash))]
-
         for i in range(max(1, threadCount)):
             t = threading.Thread(name = 'thread%s' % i, target = self.pow, args = (True,))
             t.start()

src/onionrstatistics/serializeddata.py

@@ -1,8 +1,14 @@
-"""
-    Onionr - Private P2P Communication
+"""Onionr - Private P2P Communication.
 
-    This module serializes various data pieces for use in other modules, in particular the web api
+Serialize various node information
 """
+import json
+from gevent import sleep
+
+from psutil import Process
+
+from coredb import blockmetadb
+import communicator
 """
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -18,11 +24,6 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 """
 
-import json # noqa
-import time # noqa
-
-from coredb import blockmetadb # noqa
-import communicator # noqa
 
 class SerializedData:
     def __init__(self):
@@ -38,10 +39,11 @@ class SerializedData:
     def get_stats(self):
         """Return statistics about our node"""
         stats = {}
+        proc = Process()
         try:
             self._too_many
         except AttributeError:
-            time.sleep(1)
+            sleep(1)
         comm_inst = self._too_many.get(communicator.OnionrCommunicatorDaemon, args=(self._too_many,))
         connected = []
         [connected.append(x) for x in comm_inst.onlinePeers if x not in connected]
@@ -49,4 +51,5 @@ class SerializedData:
         stats['connectedNodes'] = '\n'.join(connected)
         stats['blockCount'] = len(blockmetadb.get_block_list())
         stats['blockQueueCount'] = len(comm_inst.blockQueue)
+        stats['threads'] = proc.num_threads()
         return json.dumps(stats)

src/onionrstatistics/transports/tor/__init__.py

@@ -3,7 +3,6 @@
 
 """
 import json
-from gevent import sleep
 
 from stem import CircStatus
 
@@ -29,6 +28,7 @@ class TorStats:
     def __init__(self):
         self.circuits = {}
         self.json_data = ""
+        self.controller = None
 
     def get_json(self):
         """Refresh circuits then serialize them into form:
@@ -36,6 +36,8 @@ class TorStats:
         "nodes": list of tuples containing fingerprint and nickname strings"
         "purpose": https://stem.torproject.org/api/control.html#stem.CircPurpose
         """
+        if self.controller is None:
+            self.controller = get_controller()
         self.get_circuits()
         json_serialized = {}
         for circuit in self.circuits.keys():
@@ -52,7 +54,7 @@ class TorStats:
     def get_circuits(self):
         """Update the circuit dictionary"""
         circuits = {}
-        for circ in list(sorted(get_controller().get_circuits())):
+        for circ in list(sorted(self.controller.get_circuits())):
             if circ.status != CircStatus.BUILT:
                 continue
             circuits[circ.id] = (circ.path, circ.purpose)

src/runtests/__init__.py

@@ -62,10 +62,10 @@ class OnionrRunTestManager:
             for i in RUN_TESTS:
                 last = i
                 logger.info("[RUNTIME TEST] " + last.__name__ + " started",
-                            terminal=True)
+                            terminal=True, timestamp=True)
                 i(self)
                 logger.info("[RUNTIME TEST] " + last.__name__ + " passed",
-                            terminal=True)
+                            terminal=True, timestamp=True)
         except (ValueError, AttributeError):
             logger.error(last.__name__ + ' failed assertions', terminal=True)
         except Exception as e:

src/runtests/clearnettor.py

@@ -2,13 +2,10 @@
 
 Ensure that clearnet cannot be reached
 """
-from threading import Thread
-
 from onionrutils.basicrequests import do_get_request
 from onionrutils import localcommand
 import logger
 import config
-
 """
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by

src/runtests/lanservertest.py

@@ -10,11 +10,15 @@ def test_lan_server(testmanager):
         try:
             if requests.get(f"http://{best_ip}:{i}/ping").text == 'pong!':
                 bl = insert('test data')
-                if bl not in requests.get(f"http://{best_ip}:{i}/blist/0").text:
+                bl2 = insert('test data2')
+                l = requests.get(f"http://{best_ip}:{i}/blist/0").text
+                if bl not in l or bl2 not in l:
                     raise ValueError
                 if onionrblockapi.Block(bl).raw != requests.get(f"http://{best_ip}:{i}/get/{bl}").content:
                     raise ValueError
+                
                 break
+
         except requests.exceptions.ConnectionError:
             pass
     else:

tests/runtime-result.txt

@@ -1 +1 @@
-1584597517
+1584652661