progress removing onionr.py

onionr/apiservers/private/__init__.py

@@ -23,7 +23,7 @@ from gevent.pywsgi import WSGIServer
 from onionrutils import epoch
 import httpapi, filepaths, logger
 from . import register_private_blueprints
-import serializeddata
+import serializeddata, config
 class PrivateAPI:
     '''
         Client HTTP api
@@ -31,17 +31,15 @@ class PrivateAPI:
 
     callbacks = {'public' : {}, 'private' : {}}
 
-    def __init__(self, onionrInst, debug, API_VERSION):
+    def __init__(self):
         '''
             Initialize the api server, preping variables for later use
 
             This initialization defines all of the API entry points and handlers for the endpoints and errors
             This also saves the used host (random localhost IP address) to the data folder in host.txt
         '''
-        config = onionrInst.config
         self.config = config
-        self.debug = debug
+        self.serializer = serializeddata.SerializedData()
-        self.serializer = serializeddata.SerializedData(onionrInst)
         self.startTime = epoch.get_epoch()
         app = flask.Flask(__name__)
         bindPort = int(config.get('client.client.port', 59496))
@@ -55,13 +53,11 @@ class PrivateAPI:
         self.host = httpapi.apiutils.setbindip.set_bind_IP(filepaths.private_API_host_file)
         logger.info('Running api on %s:%s' % (self.host, self.bindPort))
         self.httpServer = ''
-        onionrInst.setClientAPIInst(self)
 
         self.queueResponse = {}
         self.get_block_data = httpapi.apiutils.GetBlockData(self)
         register_private_blueprints.register_private_blueprints(self, app)
         httpapi.load_plugin_blueprints(app)
-        self.onionrInst = onionrInst
 
         self.httpServer = WSGIServer((self.host, bindPort), app, log=None, handler_class=httpapi.fdsafehandler.FDSafeHandler)
         self.httpServer.serve_forever()

onionr/apiservers/public/__init__.py

@@ -21,14 +21,14 @@ import time
 import flask
 from gevent.pywsgi import WSGIServer
 from httpapi import apiutils, security, fdsafehandler, miscpublicapi
-import logger, onionr, filepaths
+import logger, config, filepaths
 from utils import gettransports
+from etc import onionrvalues
 class PublicAPI:
     '''
         The new client api server, isolated from the public api
     '''
-    def __init__(self, clientAPI):
+    def __init__(self):
-        config = clientAPI.config
         app = flask.Flask('PublicAPI')
         app.config['MAX_CONTENT_LENGTH'] = 5 * 1024 * 1024
         self.i2pEnabled = config.get('i2p.host', False)
@@ -43,12 +43,9 @@ class PublicAPI:
         self.lastRequest = 0
         self.hitCount = 0 # total rec requests to public api since server started
         self.config = config
-        self.clientAPI = clientAPI
+        self.API_VERSION = onionrvalues.API_VERSION
-        self.API_VERSION = onionr.API_VERSION
         logger.info('Running public api on %s:%s' % (self.host, self.bindPort))
 
-        # Set instances, then startup our public api server
-        clientAPI.setPublicAPIInstance(self)
         
         app.register_blueprint(security.public.PublicAPISecurity(self).public_api_security_bp)
         app.register_blueprint(miscpublicapi.endpoints.PublicEndpoints(self).public_endpoints_bp)

onionr/communicator/__init__.py

@@ -19,7 +19,7 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 import sys, os, time
-import config, logger, onionr
+import config, logger
 import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block
 from . import onlinepeers
 from communicatorutils import servicecreator, onionrcommunicatortimers
@@ -29,18 +29,15 @@ from communicatorutils import daemonqueuehandler, announcenode, deniableinserts
 from communicatorutils import cooldownpeer, housekeeping, netcheck
 from onionrutils import localcommand, epoch
 from etc import humanreadabletime
-import onionrservices, onionr, filepaths, storagecounter
+import onionrservices, filepaths, storagecounter
 from coredb import daemonqueue, dbfiles
 from utils import gettransports
 OnionrCommunicatorTimers = onionrcommunicatortimers.OnionrCommunicatorTimers
 
 config.reload()
 class OnionrCommunicatorDaemon:
-    def __init__(self, onionrInst, proxyPort, developmentMode=config.get('general.dev_mode', False)):
+    def __init__(self, proxyPort, developmentMode=config.get('general.dev_mode', False)):
-        onionrInst.communicatorInst = self
         # configure logger and stuff
-        onionr.Onionr.setupConfig(onionrInst)
-        self.onionrInst = onionrInst
         self.config = config
         self.storage_counter = storagecounter.StorageCounter()
         self.proxyPort = proxyPort
@@ -213,7 +210,7 @@ class OnionrCommunicatorDaemon:
 
     def peerCleanup(self):
         '''This just calls onionrpeers.cleanupPeers, which removes dead or bad peers (offline too long, too slow)'''
-        onionrpeers.peer_cleanup(self.onionrInst)
+        onionrpeers.peer_cleanup()
         self.decrementThreadCount('peerCleanup')
 
     def getPeerProfileInstance(self, peer):
@@ -247,8 +244,8 @@ class OnionrCommunicatorDaemon:
 
         self.decrementThreadCount('runCheck')
 
-def startCommunicator(onionrInst, proxyPort):
+def startCommunicator(proxyPort):
-    OnionrCommunicatorDaemon(onionrInst, proxyPort)
+    OnionrCommunicatorDaemon(proxyPort)
 
 def run_file_exists(daemon):
     if os.path.isfile(filepaths.run_check_file):

onionr/filepaths/__init__.py

@@ -11,6 +11,7 @@ bootstrap_file_location = 'static-data/bootstrap-nodes.txt'
 data_nonce_file = home + 'block-nonces.dat'
 forward_keys_file = home + 'forward-keys.db'
 cached_storage = home + 'cachedstorage.dat'
+announce_cache = home + 'announcecache.dat'
 export_location = home + 'block-export/'
 
 tor_hs_address_file = home + 'hs/hostname'

onionr/httpapi/miscclientapi/endpoints.py

@@ -19,13 +19,12 @@
 '''
 from flask import Response, Blueprint, request, send_from_directory, abort
 from httpapi import apiutils
-import onionrcrypto
+import onionrcrypto, config
 pub_key = onionrcrypto.pub_key
 class PrivateEndpoints:
     def __init__(self, client_api):
         private_endpoints_bp = Blueprint('privateendpoints', __name__)
         self.private_endpoints_bp = private_endpoints_bp
-        config = client_api.config
 
         @private_endpoints_bp.route('/serviceactive/<pubkey>')
         def serviceActive(pubkey):

onionr/httpapi/miscpublicapi/announce.py

@@ -19,12 +19,13 @@
 '''
 import base64
 from flask import Response
+import deadsimplekv
 import logger
 from etc import onionrvalues
 from onionrutils import stringvalidators, bytesconverter
 from utils import gettransports
-import onionrcrypto as crypto
+import onionrcrypto as crypto, filepaths
-def handle_announce(clientAPI, request):
+def handle_announce(request):
     '''
     accept announcement posts, validating POW
     clientAPI should be an instance of the clientAPI server running, request is a instance of a flask request
@@ -33,6 +34,7 @@ def handle_announce(clientAPI, request):
     powHash = ''
     randomData = ''
     newNode = ''
+
     try:
         newNode = request.form['node'].encode()
     except KeyError:
@@ -54,8 +56,11 @@ def handle_announce(clientAPI, request):
                 pass
             if powHash.startswith('0' * onionrvalues.OnionrValues().announce_pow):
                 newNode = bytesconverter.bytes_to_str(newNode)
-                if stringvalidators.validate_transport(newNode) and not newNode in clientAPI.onionrInst.communicatorInst.newPeers:
+                announce_queue = deadsimplekv.DeadSimpleKV(filepaths.announce_cache)
+                if stringvalidators.validate_transport(newNode) and not newNode in announce_queue.get('new_peers'):
                     clientAPI.onionrInst.communicatorInst.newPeers.append(newNode)
+                    announce_queue.put('new_peers', announce_queue.get('new_peers').append(newNode))
+                    announce_queue.flush()
                     resp = 'Success'
             else:
                 logger.warn(newNode.decode() + ' failed to meet POW: ' + powHash)

onionr/httpapi/miscpublicapi/endpoints.py

@@ -20,10 +20,9 @@
 from flask import Response, Blueprint, request, send_from_directory, abort
 from . import getblocks, upload, announce
 from coredb import keydb
+import config
 class PublicEndpoints:
     def __init__(self, public_api):
-        client_API = public_api.clientAPI
-        config = client_API.config
 
         public_endpoints_bp = Blueprint('publicendpoints', __name__)
         self.public_endpoints_bp = public_endpoints_bp
@@ -41,12 +40,12 @@ class PublicEndpoints:
         @public_endpoints_bp.route('/getblocklist')
         def get_block_list():
             '''Get a list of blocks, optionally filtered by epoch time stamp, excluding those hidden'''
-            return getblocks.get_public_block_list(client_API, public_api, request)
+            return getblocks.get_public_block_list(public_api, request)
 
         @public_endpoints_bp.route('/getdata/<name>')
         def get_block_data(name):
             # Share data for a block if we have it and it isn't hidden
-            return getblocks.get_block_data(client_API, public_api, name)
+            return getblocks.get_block_data(public_api, name)
 
         @public_endpoints_bp.route('/www/<path:path>')
         def www_public(path):
@@ -70,7 +69,7 @@ class PublicEndpoints:
         @public_endpoints_bp.route('/announce', methods=['post'])
         def accept_announce():
             '''Accept announcements with pow token to prevent spam'''
-            resp = announce.handle_announce(client_API, request)
+            resp = announce.handle_announce(request)
             return resp
 
         @public_endpoints_bp.route('/upload', methods=['post'])
@@ -78,4 +77,4 @@ class PublicEndpoints:
             '''Accept file uploads. In the future this will be done more often than on creation 
             to speed up block sync
             '''
-            return upload.accept_upload(client_API, request)
+            return upload.accept_upload(request)

onionr/httpapi/miscpublicapi/getblocks.py

@@ -22,12 +22,13 @@ import config
 from onionrutils import bytesconverter, stringvalidators
 from coredb import blockmetadb
 from utils import reconstructhash
-def get_public_block_list(clientAPI, publicAPI, request):
+from .. import apiutils
+def get_public_block_list(publicAPI, request):
     # Provide a list of our blocks, with a date offset
     dateAdjust = request.args.get('date')
     bList = blockmetadb.get_block_list(dateRec=dateAdjust)
     share_list = ''
-    if clientAPI.config.get('general.hide_created_blocks', True):
+    if config.get('general.hide_created_blocks', True):
         for b in publicAPI.hideBlocks:
             if b in bList:
                 # Don't share blocks we created if they haven't been *uploaded* yet, makes it harder to find who created a block
@@ -36,13 +37,13 @@ def get_public_block_list(clientAPI, publicAPI, request):
         share_list += '%s\n' % (reconstructhash.deconstruct_hash(b),)
     return Response(share_list)
 
-def get_block_data(clientAPI, publicAPI, data):
+def get_block_data(publicAPI, data):
     '''data is the block hash in hex'''
     resp = ''
     if stringvalidators.validate_hash(data):
-        if not clientAPI.config.get('general.hide_created_blocks', True) or data not in publicAPI.hideBlocks:
+        if not config.get('general.hide_created_blocks', True) or data not in publicAPI.hideBlocks:
             if data in blockmetadb.get_block_list():
-                block = clientAPI.getBlockData(data, raw=True)
+                block = apiutils.GetBlockData().get_block_data(data, raw=True)
                 try:
                     block = block.encode() # Encode in case data is binary
                 except AttributeError:

onionr/httpapi/miscpublicapi/upload.py

@@ -20,7 +20,7 @@
 import sys
 from flask import Response, abort
 import blockimporter, onionrexceptions, logger
-def accept_upload(clientAPI, request):
+def accept_upload(request):
     resp = 'failure'
     try:
         data = request.form['block']

onionr/onionrcommands/daemonlaunch.py

@@ -20,6 +20,9 @@
 
 import os, time, sys, platform, sqlite3, signal
 from threading import Thread
+
+import toomanyobjs
+
 import config, apiservers, logger, communicator
 import onionrevents as events
 from netcontroller import NetController
@@ -48,8 +51,8 @@ def daemon():
         logger.debug('Runcheck file found on daemon start, deleting in advance.')
         os.remove(filepaths.run_check_file)
 
-    Thread(target=apiservers.ClientAPI, args=(onionrvalues.API_VERSION), daemon=True).start()
+    Thread(target=apiservers.ClientAPI, daemon=True).start()
-    Thread(target=apiservers.PublicAPI, args=[o_inst.getClientApi()], daemon=True).start()
+    Thread(target=apiservers.PublicAPI, daemon=True).start()
 
     apiHost = ''
     while apiHost == '':

onionr/onionrcommands/parser/__init__.py

@@ -22,7 +22,10 @@ from etc import onionrvalues
 import logger, onionrexceptions
 from . import arguments, recommend
 def register():
-    cmd = sys.argv[1]
+    try:
+        cmd = sys.argv[1]
+    except IndexError:
+        cmd = ""
 
     try:
         arguments.get_func(cmd)()

requirements.in

@@ -9,3 +9,4 @@ deadsimplekv==0.1.1
 unpaddedbase32==0.1.0
 streamedrequests==1.0.0
 jinja2==2.10.1
+toomanyobjs==0.0.0

requirements.txt

@@ -173,6 +173,8 @@ stem==1.7.1 \
     --hash=sha256:c9eaf3116cb60c15995cbd3dec3a5cbc50e9bb6e062c4d6d42201e566f498ca2
 streamedrequests==1.0.0 \
     --hash=sha256:1d9d07394804a6e1fd66bde74a804e71cab98e6920053865574a459f1cf7d3b7
+toomanyobjs==0.0.0 \
+    --hash=sha256:997e9399a33d4884eb535b6cc2c02705ed9ae1bcff56afbbf3ea81fc4ac9ab95
 unpaddedbase32==0.1.0 \
     --hash=sha256:5e4143fcaf77c9c6b4f60d18301c7570f0dac561dcf9b9aed8b5ba6ead7f218c
 urllib3==1.24.2 \