From a6fec9eefb4ffe75fd647b92abf30cd995fa689f Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 2 Aug 2019 18:00:04 -0500 Subject: [PATCH] progress removing onionr.py --- onionr/apiservers/private/__init__.py | 10 +++------- onionr/apiservers/public/__init__.py | 11 ++++------- onionr/communicator/__init__.py | 15 ++++++--------- onionr/filepaths/__init__.py | 1 + onionr/httpapi/miscclientapi/endpoints.py | 3 +-- onionr/httpapi/miscpublicapi/announce.py | 11 ++++++++--- onionr/httpapi/miscpublicapi/endpoints.py | 11 +++++------ onionr/httpapi/miscpublicapi/getblocks.py | 11 ++++++----- onionr/httpapi/miscpublicapi/upload.py | 2 +- onionr/onionrcommands/daemonlaunch.py | 7 +++++-- onionr/onionrcommands/parser/__init__.py | 5 ++++- requirements.in | 1 + requirements.txt | 2 ++ 13 files changed, 47 insertions(+), 43 deletions(-) diff --git a/onionr/apiservers/private/__init__.py b/onionr/apiservers/private/__init__.py index 2683ed6d..d2407efb 100644 --- a/onionr/apiservers/private/__init__.py +++ b/onionr/apiservers/private/__init__.py @@ -23,7 +23,7 @@ from gevent.pywsgi import WSGIServer from onionrutils import epoch import httpapi, filepaths, logger from . import register_private_blueprints -import serializeddata +import serializeddata, config class PrivateAPI: ''' Client HTTP api @@ -31,17 +31,15 @@ class PrivateAPI: callbacks = {'public' : {}, 'private' : {}} - def __init__(self, onionrInst, debug, API_VERSION): + def __init__(self): ''' Initialize the api server, preping variables for later use This initialization defines all of the API entry points and handlers for the endpoints and errors This also saves the used host (random localhost IP address) to the data folder in host.txt ''' - config = onionrInst.config self.config = config - self.debug = debug - self.serializer = serializeddata.SerializedData(onionrInst) + self.serializer = serializeddata.SerializedData() self.startTime = epoch.get_epoch() app = flask.Flask(__name__) bindPort = int(config.get('client.client.port', 59496)) @@ -55,13 +53,11 @@ class PrivateAPI: self.host = httpapi.apiutils.setbindip.set_bind_IP(filepaths.private_API_host_file) logger.info('Running api on %s:%s' % (self.host, self.bindPort)) self.httpServer = '' - onionrInst.setClientAPIInst(self) self.queueResponse = {} self.get_block_data = httpapi.apiutils.GetBlockData(self) register_private_blueprints.register_private_blueprints(self, app) httpapi.load_plugin_blueprints(app) - self.onionrInst = onionrInst self.httpServer = WSGIServer((self.host, bindPort), app, log=None, handler_class=httpapi.fdsafehandler.FDSafeHandler) self.httpServer.serve_forever() diff --git a/onionr/apiservers/public/__init__.py b/onionr/apiservers/public/__init__.py index 4b73fcd3..ddf6383d 100644 --- a/onionr/apiservers/public/__init__.py +++ b/onionr/apiservers/public/__init__.py @@ -21,14 +21,14 @@ import time import flask from gevent.pywsgi import WSGIServer from httpapi import apiutils, security, fdsafehandler, miscpublicapi -import logger, onionr, filepaths +import logger, config, filepaths from utils import gettransports +from etc import onionrvalues class PublicAPI: ''' The new client api server, isolated from the public api ''' - def __init__(self, clientAPI): - config = clientAPI.config + def __init__(self): app = flask.Flask('PublicAPI') app.config['MAX_CONTENT_LENGTH'] = 5 * 1024 * 1024 self.i2pEnabled = config.get('i2p.host', False) @@ -43,12 +43,9 @@ class PublicAPI: self.lastRequest = 0 self.hitCount = 0 # total rec requests to public api since server started self.config = config - self.clientAPI = clientAPI - self.API_VERSION = onionr.API_VERSION + self.API_VERSION = onionrvalues.API_VERSION logger.info('Running public api on %s:%s' % (self.host, self.bindPort)) - # Set instances, then startup our public api server - clientAPI.setPublicAPIInstance(self) app.register_blueprint(security.public.PublicAPISecurity(self).public_api_security_bp) app.register_blueprint(miscpublicapi.endpoints.PublicEndpoints(self).public_endpoints_bp) diff --git a/onionr/communicator/__init__.py b/onionr/communicator/__init__.py index 8ca03a19..2aa2b373 100755 --- a/onionr/communicator/__init__.py +++ b/onionr/communicator/__init__.py @@ -19,7 +19,7 @@ along with this program. If not, see . ''' import sys, os, time -import config, logger, onionr +import config, logger import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block from . import onlinepeers from communicatorutils import servicecreator, onionrcommunicatortimers @@ -29,18 +29,15 @@ from communicatorutils import daemonqueuehandler, announcenode, deniableinserts from communicatorutils import cooldownpeer, housekeeping, netcheck from onionrutils import localcommand, epoch from etc import humanreadabletime -import onionrservices, onionr, filepaths, storagecounter +import onionrservices, filepaths, storagecounter from coredb import daemonqueue, dbfiles from utils import gettransports OnionrCommunicatorTimers = onionrcommunicatortimers.OnionrCommunicatorTimers config.reload() class OnionrCommunicatorDaemon: - def __init__(self, onionrInst, proxyPort, developmentMode=config.get('general.dev_mode', False)): - onionrInst.communicatorInst = self + def __init__(self, proxyPort, developmentMode=config.get('general.dev_mode', False)): # configure logger and stuff - onionr.Onionr.setupConfig(onionrInst) - self.onionrInst = onionrInst self.config = config self.storage_counter = storagecounter.StorageCounter() self.proxyPort = proxyPort @@ -213,7 +210,7 @@ class OnionrCommunicatorDaemon: def peerCleanup(self): '''This just calls onionrpeers.cleanupPeers, which removes dead or bad peers (offline too long, too slow)''' - onionrpeers.peer_cleanup(self.onionrInst) + onionrpeers.peer_cleanup() self.decrementThreadCount('peerCleanup') def getPeerProfileInstance(self, peer): @@ -247,8 +244,8 @@ class OnionrCommunicatorDaemon: self.decrementThreadCount('runCheck') -def startCommunicator(onionrInst, proxyPort): - OnionrCommunicatorDaemon(onionrInst, proxyPort) +def startCommunicator(proxyPort): + OnionrCommunicatorDaemon(proxyPort) def run_file_exists(daemon): if os.path.isfile(filepaths.run_check_file): diff --git a/onionr/filepaths/__init__.py b/onionr/filepaths/__init__.py index 2d839c10..14006e90 100644 --- a/onionr/filepaths/__init__.py +++ b/onionr/filepaths/__init__.py @@ -11,6 +11,7 @@ bootstrap_file_location = 'static-data/bootstrap-nodes.txt' data_nonce_file = home + 'block-nonces.dat' forward_keys_file = home + 'forward-keys.db' cached_storage = home + 'cachedstorage.dat' +announce_cache = home + 'announcecache.dat' export_location = home + 'block-export/' tor_hs_address_file = home + 'hs/hostname' diff --git a/onionr/httpapi/miscclientapi/endpoints.py b/onionr/httpapi/miscclientapi/endpoints.py index f0c86204..ca20abec 100644 --- a/onionr/httpapi/miscclientapi/endpoints.py +++ b/onionr/httpapi/miscclientapi/endpoints.py @@ -19,13 +19,12 @@ ''' from flask import Response, Blueprint, request, send_from_directory, abort from httpapi import apiutils -import onionrcrypto +import onionrcrypto, config pub_key = onionrcrypto.pub_key class PrivateEndpoints: def __init__(self, client_api): private_endpoints_bp = Blueprint('privateendpoints', __name__) self.private_endpoints_bp = private_endpoints_bp - config = client_api.config @private_endpoints_bp.route('/serviceactive/') def serviceActive(pubkey): diff --git a/onionr/httpapi/miscpublicapi/announce.py b/onionr/httpapi/miscpublicapi/announce.py index 8009704a..8e477630 100755 --- a/onionr/httpapi/miscpublicapi/announce.py +++ b/onionr/httpapi/miscpublicapi/announce.py @@ -19,12 +19,13 @@ ''' import base64 from flask import Response +import deadsimplekv import logger from etc import onionrvalues from onionrutils import stringvalidators, bytesconverter from utils import gettransports -import onionrcrypto as crypto -def handle_announce(clientAPI, request): +import onionrcrypto as crypto, filepaths +def handle_announce(request): ''' accept announcement posts, validating POW clientAPI should be an instance of the clientAPI server running, request is a instance of a flask request @@ -33,6 +34,7 @@ def handle_announce(clientAPI, request): powHash = '' randomData = '' newNode = '' + try: newNode = request.form['node'].encode() except KeyError: @@ -54,8 +56,11 @@ def handle_announce(clientAPI, request): pass if powHash.startswith('0' * onionrvalues.OnionrValues().announce_pow): newNode = bytesconverter.bytes_to_str(newNode) - if stringvalidators.validate_transport(newNode) and not newNode in clientAPI.onionrInst.communicatorInst.newPeers: + announce_queue = deadsimplekv.DeadSimpleKV(filepaths.announce_cache) + if stringvalidators.validate_transport(newNode) and not newNode in announce_queue.get('new_peers'): clientAPI.onionrInst.communicatorInst.newPeers.append(newNode) + announce_queue.put('new_peers', announce_queue.get('new_peers').append(newNode)) + announce_queue.flush() resp = 'Success' else: logger.warn(newNode.decode() + ' failed to meet POW: ' + powHash) diff --git a/onionr/httpapi/miscpublicapi/endpoints.py b/onionr/httpapi/miscpublicapi/endpoints.py index 7a5db32b..7318948d 100644 --- a/onionr/httpapi/miscpublicapi/endpoints.py +++ b/onionr/httpapi/miscpublicapi/endpoints.py @@ -20,10 +20,9 @@ from flask import Response, Blueprint, request, send_from_directory, abort from . import getblocks, upload, announce from coredb import keydb +import config class PublicEndpoints: def __init__(self, public_api): - client_API = public_api.clientAPI - config = client_API.config public_endpoints_bp = Blueprint('publicendpoints', __name__) self.public_endpoints_bp = public_endpoints_bp @@ -41,12 +40,12 @@ class PublicEndpoints: @public_endpoints_bp.route('/getblocklist') def get_block_list(): '''Get a list of blocks, optionally filtered by epoch time stamp, excluding those hidden''' - return getblocks.get_public_block_list(client_API, public_api, request) + return getblocks.get_public_block_list(public_api, request) @public_endpoints_bp.route('/getdata/') def get_block_data(name): # Share data for a block if we have it and it isn't hidden - return getblocks.get_block_data(client_API, public_api, name) + return getblocks.get_block_data(public_api, name) @public_endpoints_bp.route('/www/') def www_public(path): @@ -70,7 +69,7 @@ class PublicEndpoints: @public_endpoints_bp.route('/announce', methods=['post']) def accept_announce(): '''Accept announcements with pow token to prevent spam''' - resp = announce.handle_announce(client_API, request) + resp = announce.handle_announce(request) return resp @public_endpoints_bp.route('/upload', methods=['post']) @@ -78,4 +77,4 @@ class PublicEndpoints: '''Accept file uploads. In the future this will be done more often than on creation to speed up block sync ''' - return upload.accept_upload(client_API, request) \ No newline at end of file + return upload.accept_upload(request) \ No newline at end of file diff --git a/onionr/httpapi/miscpublicapi/getblocks.py b/onionr/httpapi/miscpublicapi/getblocks.py index 306c0a34..28693110 100755 --- a/onionr/httpapi/miscpublicapi/getblocks.py +++ b/onionr/httpapi/miscpublicapi/getblocks.py @@ -22,12 +22,13 @@ import config from onionrutils import bytesconverter, stringvalidators from coredb import blockmetadb from utils import reconstructhash -def get_public_block_list(clientAPI, publicAPI, request): +from .. import apiutils +def get_public_block_list(publicAPI, request): # Provide a list of our blocks, with a date offset dateAdjust = request.args.get('date') bList = blockmetadb.get_block_list(dateRec=dateAdjust) share_list = '' - if clientAPI.config.get('general.hide_created_blocks', True): + if config.get('general.hide_created_blocks', True): for b in publicAPI.hideBlocks: if b in bList: # Don't share blocks we created if they haven't been *uploaded* yet, makes it harder to find who created a block @@ -36,13 +37,13 @@ def get_public_block_list(clientAPI, publicAPI, request): share_list += '%s\n' % (reconstructhash.deconstruct_hash(b),) return Response(share_list) -def get_block_data(clientAPI, publicAPI, data): +def get_block_data(publicAPI, data): '''data is the block hash in hex''' resp = '' if stringvalidators.validate_hash(data): - if not clientAPI.config.get('general.hide_created_blocks', True) or data not in publicAPI.hideBlocks: + if not config.get('general.hide_created_blocks', True) or data not in publicAPI.hideBlocks: if data in blockmetadb.get_block_list(): - block = clientAPI.getBlockData(data, raw=True) + block = apiutils.GetBlockData().get_block_data(data, raw=True) try: block = block.encode() # Encode in case data is binary except AttributeError: diff --git a/onionr/httpapi/miscpublicapi/upload.py b/onionr/httpapi/miscpublicapi/upload.py index bc8ceac9..87811fbe 100755 --- a/onionr/httpapi/miscpublicapi/upload.py +++ b/onionr/httpapi/miscpublicapi/upload.py @@ -20,7 +20,7 @@ import sys from flask import Response, abort import blockimporter, onionrexceptions, logger -def accept_upload(clientAPI, request): +def accept_upload(request): resp = 'failure' try: data = request.form['block'] diff --git a/onionr/onionrcommands/daemonlaunch.py b/onionr/onionrcommands/daemonlaunch.py index 15b9e1ce..1ad14bd0 100755 --- a/onionr/onionrcommands/daemonlaunch.py +++ b/onionr/onionrcommands/daemonlaunch.py @@ -20,6 +20,9 @@ import os, time, sys, platform, sqlite3, signal from threading import Thread + +import toomanyobjs + import config, apiservers, logger, communicator import onionrevents as events from netcontroller import NetController @@ -48,8 +51,8 @@ def daemon(): logger.debug('Runcheck file found on daemon start, deleting in advance.') os.remove(filepaths.run_check_file) - Thread(target=apiservers.ClientAPI, args=(onionrvalues.API_VERSION), daemon=True).start() - Thread(target=apiservers.PublicAPI, args=[o_inst.getClientApi()], daemon=True).start() + Thread(target=apiservers.ClientAPI, daemon=True).start() + Thread(target=apiservers.PublicAPI, daemon=True).start() apiHost = '' while apiHost == '': diff --git a/onionr/onionrcommands/parser/__init__.py b/onionr/onionrcommands/parser/__init__.py index d5d1bebe..2f14f080 100644 --- a/onionr/onionrcommands/parser/__init__.py +++ b/onionr/onionrcommands/parser/__init__.py @@ -22,7 +22,10 @@ from etc import onionrvalues import logger, onionrexceptions from . import arguments, recommend def register(): - cmd = sys.argv[1] + try: + cmd = sys.argv[1] + except IndexError: + cmd = "" try: arguments.get_func(cmd)() diff --git a/requirements.in b/requirements.in index 3b90933e..fcdffa71 100644 --- a/requirements.in +++ b/requirements.in @@ -9,3 +9,4 @@ deadsimplekv==0.1.1 unpaddedbase32==0.1.0 streamedrequests==1.0.0 jinja2==2.10.1 +toomanyobjs==0.0.0 diff --git a/requirements.txt b/requirements.txt index b5ca0af4..b511c9ab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -173,6 +173,8 @@ stem==1.7.1 \ --hash=sha256:c9eaf3116cb60c15995cbd3dec3a5cbc50e9bb6e062c4d6d42201e566f498ca2 streamedrequests==1.0.0 \ --hash=sha256:1d9d07394804a6e1fd66bde74a804e71cab98e6920053865574a459f1cf7d3b7 +toomanyobjs==0.0.0 \ + --hash=sha256:997e9399a33d4884eb535b6cc2c02705ed9ae1bcff56afbbf3ea81fc4ac9ab95 unpaddedbase32==0.1.0 \ --hash=sha256:5e4143fcaf77c9c6b4f60d18301c7570f0dac561dcf9b9aed8b5ba6ead7f218c urllib3==1.24.2 \