progress removing onionr.py

This commit is contained in:
Kevin Froman 2019-08-02 18:00:04 -05:00
parent 6e526bf629
commit a6fec9eefb
13 changed files with 47 additions and 43 deletions

View File

@ -23,7 +23,7 @@ from gevent.pywsgi import WSGIServer
from onionrutils import epoch from onionrutils import epoch
import httpapi, filepaths, logger import httpapi, filepaths, logger
from . import register_private_blueprints from . import register_private_blueprints
import serializeddata import serializeddata, config
class PrivateAPI: class PrivateAPI:
''' '''
Client HTTP api Client HTTP api
@ -31,17 +31,15 @@ class PrivateAPI:
callbacks = {'public' : {}, 'private' : {}} callbacks = {'public' : {}, 'private' : {}}
def __init__(self, onionrInst, debug, API_VERSION): def __init__(self):
''' '''
Initialize the api server, preping variables for later use Initialize the api server, preping variables for later use
This initialization defines all of the API entry points and handlers for the endpoints and errors This initialization defines all of the API entry points and handlers for the endpoints and errors
This also saves the used host (random localhost IP address) to the data folder in host.txt This also saves the used host (random localhost IP address) to the data folder in host.txt
''' '''
config = onionrInst.config
self.config = config self.config = config
self.debug = debug self.serializer = serializeddata.SerializedData()
self.serializer = serializeddata.SerializedData(onionrInst)
self.startTime = epoch.get_epoch() self.startTime = epoch.get_epoch()
app = flask.Flask(__name__) app = flask.Flask(__name__)
bindPort = int(config.get('client.client.port', 59496)) bindPort = int(config.get('client.client.port', 59496))
@ -55,13 +53,11 @@ class PrivateAPI:
self.host = httpapi.apiutils.setbindip.set_bind_IP(filepaths.private_API_host_file) self.host = httpapi.apiutils.setbindip.set_bind_IP(filepaths.private_API_host_file)
logger.info('Running api on %s:%s' % (self.host, self.bindPort)) logger.info('Running api on %s:%s' % (self.host, self.bindPort))
self.httpServer = '' self.httpServer = ''
onionrInst.setClientAPIInst(self)
self.queueResponse = {} self.queueResponse = {}
self.get_block_data = httpapi.apiutils.GetBlockData(self) self.get_block_data = httpapi.apiutils.GetBlockData(self)
register_private_blueprints.register_private_blueprints(self, app) register_private_blueprints.register_private_blueprints(self, app)
httpapi.load_plugin_blueprints(app) httpapi.load_plugin_blueprints(app)
self.onionrInst = onionrInst
self.httpServer = WSGIServer((self.host, bindPort), app, log=None, handler_class=httpapi.fdsafehandler.FDSafeHandler) self.httpServer = WSGIServer((self.host, bindPort), app, log=None, handler_class=httpapi.fdsafehandler.FDSafeHandler)
self.httpServer.serve_forever() self.httpServer.serve_forever()

View File

@ -21,14 +21,14 @@ import time
import flask import flask
from gevent.pywsgi import WSGIServer from gevent.pywsgi import WSGIServer
from httpapi import apiutils, security, fdsafehandler, miscpublicapi from httpapi import apiutils, security, fdsafehandler, miscpublicapi
import logger, onionr, filepaths import logger, config, filepaths
from utils import gettransports from utils import gettransports
from etc import onionrvalues
class PublicAPI: class PublicAPI:
''' '''
The new client api server, isolated from the public api The new client api server, isolated from the public api
''' '''
def __init__(self, clientAPI): def __init__(self):
config = clientAPI.config
app = flask.Flask('PublicAPI') app = flask.Flask('PublicAPI')
app.config['MAX_CONTENT_LENGTH'] = 5 * 1024 * 1024 app.config['MAX_CONTENT_LENGTH'] = 5 * 1024 * 1024
self.i2pEnabled = config.get('i2p.host', False) self.i2pEnabled = config.get('i2p.host', False)
@ -43,12 +43,9 @@ class PublicAPI:
self.lastRequest = 0 self.lastRequest = 0
self.hitCount = 0 # total rec requests to public api since server started self.hitCount = 0 # total rec requests to public api since server started
self.config = config self.config = config
self.clientAPI = clientAPI self.API_VERSION = onionrvalues.API_VERSION
self.API_VERSION = onionr.API_VERSION
logger.info('Running public api on %s:%s' % (self.host, self.bindPort)) logger.info('Running public api on %s:%s' % (self.host, self.bindPort))
# Set instances, then startup our public api server
clientAPI.setPublicAPIInstance(self)
app.register_blueprint(security.public.PublicAPISecurity(self).public_api_security_bp) app.register_blueprint(security.public.PublicAPISecurity(self).public_api_security_bp)
app.register_blueprint(miscpublicapi.endpoints.PublicEndpoints(self).public_endpoints_bp) app.register_blueprint(miscpublicapi.endpoints.PublicEndpoints(self).public_endpoints_bp)

View File

@ -19,7 +19,7 @@
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
import sys, os, time import sys, os, time
import config, logger, onionr import config, logger
import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block
from . import onlinepeers from . import onlinepeers
from communicatorutils import servicecreator, onionrcommunicatortimers from communicatorutils import servicecreator, onionrcommunicatortimers
@ -29,18 +29,15 @@ from communicatorutils import daemonqueuehandler, announcenode, deniableinserts
from communicatorutils import cooldownpeer, housekeeping, netcheck from communicatorutils import cooldownpeer, housekeeping, netcheck
from onionrutils import localcommand, epoch from onionrutils import localcommand, epoch
from etc import humanreadabletime from etc import humanreadabletime
import onionrservices, onionr, filepaths, storagecounter import onionrservices, filepaths, storagecounter
from coredb import daemonqueue, dbfiles from coredb import daemonqueue, dbfiles
from utils import gettransports from utils import gettransports
OnionrCommunicatorTimers = onionrcommunicatortimers.OnionrCommunicatorTimers OnionrCommunicatorTimers = onionrcommunicatortimers.OnionrCommunicatorTimers
config.reload() config.reload()
class OnionrCommunicatorDaemon: class OnionrCommunicatorDaemon:
def __init__(self, onionrInst, proxyPort, developmentMode=config.get('general.dev_mode', False)): def __init__(self, proxyPort, developmentMode=config.get('general.dev_mode', False)):
onionrInst.communicatorInst = self
# configure logger and stuff # configure logger and stuff
onionr.Onionr.setupConfig(onionrInst)
self.onionrInst = onionrInst
self.config = config self.config = config
self.storage_counter = storagecounter.StorageCounter() self.storage_counter = storagecounter.StorageCounter()
self.proxyPort = proxyPort self.proxyPort = proxyPort
@ -213,7 +210,7 @@ class OnionrCommunicatorDaemon:
def peerCleanup(self): def peerCleanup(self):
'''This just calls onionrpeers.cleanupPeers, which removes dead or bad peers (offline too long, too slow)''' '''This just calls onionrpeers.cleanupPeers, which removes dead or bad peers (offline too long, too slow)'''
onionrpeers.peer_cleanup(self.onionrInst) onionrpeers.peer_cleanup()
self.decrementThreadCount('peerCleanup') self.decrementThreadCount('peerCleanup')
def getPeerProfileInstance(self, peer): def getPeerProfileInstance(self, peer):
@ -247,8 +244,8 @@ class OnionrCommunicatorDaemon:
self.decrementThreadCount('runCheck') self.decrementThreadCount('runCheck')
def startCommunicator(onionrInst, proxyPort): def startCommunicator(proxyPort):
OnionrCommunicatorDaemon(onionrInst, proxyPort) OnionrCommunicatorDaemon(proxyPort)
def run_file_exists(daemon): def run_file_exists(daemon):
if os.path.isfile(filepaths.run_check_file): if os.path.isfile(filepaths.run_check_file):

View File

@ -11,6 +11,7 @@ bootstrap_file_location = 'static-data/bootstrap-nodes.txt'
data_nonce_file = home + 'block-nonces.dat' data_nonce_file = home + 'block-nonces.dat'
forward_keys_file = home + 'forward-keys.db' forward_keys_file = home + 'forward-keys.db'
cached_storage = home + 'cachedstorage.dat' cached_storage = home + 'cachedstorage.dat'
announce_cache = home + 'announcecache.dat'
export_location = home + 'block-export/' export_location = home + 'block-export/'
tor_hs_address_file = home + 'hs/hostname' tor_hs_address_file = home + 'hs/hostname'

View File

@ -19,13 +19,12 @@
''' '''
from flask import Response, Blueprint, request, send_from_directory, abort from flask import Response, Blueprint, request, send_from_directory, abort
from httpapi import apiutils from httpapi import apiutils
import onionrcrypto import onionrcrypto, config
pub_key = onionrcrypto.pub_key pub_key = onionrcrypto.pub_key
class PrivateEndpoints: class PrivateEndpoints:
def __init__(self, client_api): def __init__(self, client_api):
private_endpoints_bp = Blueprint('privateendpoints', __name__) private_endpoints_bp = Blueprint('privateendpoints', __name__)
self.private_endpoints_bp = private_endpoints_bp self.private_endpoints_bp = private_endpoints_bp
config = client_api.config
@private_endpoints_bp.route('/serviceactive/<pubkey>') @private_endpoints_bp.route('/serviceactive/<pubkey>')
def serviceActive(pubkey): def serviceActive(pubkey):

View File

@ -19,12 +19,13 @@
''' '''
import base64 import base64
from flask import Response from flask import Response
import deadsimplekv
import logger import logger
from etc import onionrvalues from etc import onionrvalues
from onionrutils import stringvalidators, bytesconverter from onionrutils import stringvalidators, bytesconverter
from utils import gettransports from utils import gettransports
import onionrcrypto as crypto import onionrcrypto as crypto, filepaths
def handle_announce(clientAPI, request): def handle_announce(request):
''' '''
accept announcement posts, validating POW accept announcement posts, validating POW
clientAPI should be an instance of the clientAPI server running, request is a instance of a flask request clientAPI should be an instance of the clientAPI server running, request is a instance of a flask request
@ -33,6 +34,7 @@ def handle_announce(clientAPI, request):
powHash = '' powHash = ''
randomData = '' randomData = ''
newNode = '' newNode = ''
try: try:
newNode = request.form['node'].encode() newNode = request.form['node'].encode()
except KeyError: except KeyError:
@ -54,8 +56,11 @@ def handle_announce(clientAPI, request):
pass pass
if powHash.startswith('0' * onionrvalues.OnionrValues().announce_pow): if powHash.startswith('0' * onionrvalues.OnionrValues().announce_pow):
newNode = bytesconverter.bytes_to_str(newNode) newNode = bytesconverter.bytes_to_str(newNode)
if stringvalidators.validate_transport(newNode) and not newNode in clientAPI.onionrInst.communicatorInst.newPeers: announce_queue = deadsimplekv.DeadSimpleKV(filepaths.announce_cache)
if stringvalidators.validate_transport(newNode) and not newNode in announce_queue.get('new_peers'):
clientAPI.onionrInst.communicatorInst.newPeers.append(newNode) clientAPI.onionrInst.communicatorInst.newPeers.append(newNode)
announce_queue.put('new_peers', announce_queue.get('new_peers').append(newNode))
announce_queue.flush()
resp = 'Success' resp = 'Success'
else: else:
logger.warn(newNode.decode() + ' failed to meet POW: ' + powHash) logger.warn(newNode.decode() + ' failed to meet POW: ' + powHash)

View File

@ -20,10 +20,9 @@
from flask import Response, Blueprint, request, send_from_directory, abort from flask import Response, Blueprint, request, send_from_directory, abort
from . import getblocks, upload, announce from . import getblocks, upload, announce
from coredb import keydb from coredb import keydb
import config
class PublicEndpoints: class PublicEndpoints:
def __init__(self, public_api): def __init__(self, public_api):
client_API = public_api.clientAPI
config = client_API.config
public_endpoints_bp = Blueprint('publicendpoints', __name__) public_endpoints_bp = Blueprint('publicendpoints', __name__)
self.public_endpoints_bp = public_endpoints_bp self.public_endpoints_bp = public_endpoints_bp
@ -41,12 +40,12 @@ class PublicEndpoints:
@public_endpoints_bp.route('/getblocklist') @public_endpoints_bp.route('/getblocklist')
def get_block_list(): def get_block_list():
'''Get a list of blocks, optionally filtered by epoch time stamp, excluding those hidden''' '''Get a list of blocks, optionally filtered by epoch time stamp, excluding those hidden'''
return getblocks.get_public_block_list(client_API, public_api, request) return getblocks.get_public_block_list(public_api, request)
@public_endpoints_bp.route('/getdata/<name>') @public_endpoints_bp.route('/getdata/<name>')
def get_block_data(name): def get_block_data(name):
# Share data for a block if we have it and it isn't hidden # Share data for a block if we have it and it isn't hidden
return getblocks.get_block_data(client_API, public_api, name) return getblocks.get_block_data(public_api, name)
@public_endpoints_bp.route('/www/<path:path>') @public_endpoints_bp.route('/www/<path:path>')
def www_public(path): def www_public(path):
@ -70,7 +69,7 @@ class PublicEndpoints:
@public_endpoints_bp.route('/announce', methods=['post']) @public_endpoints_bp.route('/announce', methods=['post'])
def accept_announce(): def accept_announce():
'''Accept announcements with pow token to prevent spam''' '''Accept announcements with pow token to prevent spam'''
resp = announce.handle_announce(client_API, request) resp = announce.handle_announce(request)
return resp return resp
@public_endpoints_bp.route('/upload', methods=['post']) @public_endpoints_bp.route('/upload', methods=['post'])
@ -78,4 +77,4 @@ class PublicEndpoints:
'''Accept file uploads. In the future this will be done more often than on creation '''Accept file uploads. In the future this will be done more often than on creation
to speed up block sync to speed up block sync
''' '''
return upload.accept_upload(client_API, request) return upload.accept_upload(request)

View File

@ -22,12 +22,13 @@ import config
from onionrutils import bytesconverter, stringvalidators from onionrutils import bytesconverter, stringvalidators
from coredb import blockmetadb from coredb import blockmetadb
from utils import reconstructhash from utils import reconstructhash
def get_public_block_list(clientAPI, publicAPI, request): from .. import apiutils
def get_public_block_list(publicAPI, request):
# Provide a list of our blocks, with a date offset # Provide a list of our blocks, with a date offset
dateAdjust = request.args.get('date') dateAdjust = request.args.get('date')
bList = blockmetadb.get_block_list(dateRec=dateAdjust) bList = blockmetadb.get_block_list(dateRec=dateAdjust)
share_list = '' share_list = ''
if clientAPI.config.get('general.hide_created_blocks', True): if config.get('general.hide_created_blocks', True):
for b in publicAPI.hideBlocks: for b in publicAPI.hideBlocks:
if b in bList: if b in bList:
# Don't share blocks we created if they haven't been *uploaded* yet, makes it harder to find who created a block # Don't share blocks we created if they haven't been *uploaded* yet, makes it harder to find who created a block
@ -36,13 +37,13 @@ def get_public_block_list(clientAPI, publicAPI, request):
share_list += '%s\n' % (reconstructhash.deconstruct_hash(b),) share_list += '%s\n' % (reconstructhash.deconstruct_hash(b),)
return Response(share_list) return Response(share_list)
def get_block_data(clientAPI, publicAPI, data): def get_block_data(publicAPI, data):
'''data is the block hash in hex''' '''data is the block hash in hex'''
resp = '' resp = ''
if stringvalidators.validate_hash(data): if stringvalidators.validate_hash(data):
if not clientAPI.config.get('general.hide_created_blocks', True) or data not in publicAPI.hideBlocks: if not config.get('general.hide_created_blocks', True) or data not in publicAPI.hideBlocks:
if data in blockmetadb.get_block_list(): if data in blockmetadb.get_block_list():
block = clientAPI.getBlockData(data, raw=True) block = apiutils.GetBlockData().get_block_data(data, raw=True)
try: try:
block = block.encode() # Encode in case data is binary block = block.encode() # Encode in case data is binary
except AttributeError: except AttributeError:

View File

@ -20,7 +20,7 @@
import sys import sys
from flask import Response, abort from flask import Response, abort
import blockimporter, onionrexceptions, logger import blockimporter, onionrexceptions, logger
def accept_upload(clientAPI, request): def accept_upload(request):
resp = 'failure' resp = 'failure'
try: try:
data = request.form['block'] data = request.form['block']

View File

@ -20,6 +20,9 @@
import os, time, sys, platform, sqlite3, signal import os, time, sys, platform, sqlite3, signal
from threading import Thread from threading import Thread
import toomanyobjs
import config, apiservers, logger, communicator import config, apiservers, logger, communicator
import onionrevents as events import onionrevents as events
from netcontroller import NetController from netcontroller import NetController
@ -48,8 +51,8 @@ def daemon():
logger.debug('Runcheck file found on daemon start, deleting in advance.') logger.debug('Runcheck file found on daemon start, deleting in advance.')
os.remove(filepaths.run_check_file) os.remove(filepaths.run_check_file)
Thread(target=apiservers.ClientAPI, args=(onionrvalues.API_VERSION), daemon=True).start() Thread(target=apiservers.ClientAPI, daemon=True).start()
Thread(target=apiservers.PublicAPI, args=[o_inst.getClientApi()], daemon=True).start() Thread(target=apiservers.PublicAPI, daemon=True).start()
apiHost = '' apiHost = ''
while apiHost == '': while apiHost == '':

View File

@ -22,7 +22,10 @@ from etc import onionrvalues
import logger, onionrexceptions import logger, onionrexceptions
from . import arguments, recommend from . import arguments, recommend
def register(): def register():
cmd = sys.argv[1] try:
cmd = sys.argv[1]
except IndexError:
cmd = ""
try: try:
arguments.get_func(cmd)() arguments.get_func(cmd)()

View File

@ -9,3 +9,4 @@ deadsimplekv==0.1.1
unpaddedbase32==0.1.0 unpaddedbase32==0.1.0
streamedrequests==1.0.0 streamedrequests==1.0.0
jinja2==2.10.1 jinja2==2.10.1
toomanyobjs==0.0.0

View File

@ -173,6 +173,8 @@ stem==1.7.1 \
--hash=sha256:c9eaf3116cb60c15995cbd3dec3a5cbc50e9bb6e062c4d6d42201e566f498ca2 --hash=sha256:c9eaf3116cb60c15995cbd3dec3a5cbc50e9bb6e062c4d6d42201e566f498ca2
streamedrequests==1.0.0 \ streamedrequests==1.0.0 \
--hash=sha256:1d9d07394804a6e1fd66bde74a804e71cab98e6920053865574a459f1cf7d3b7 --hash=sha256:1d9d07394804a6e1fd66bde74a804e71cab98e6920053865574a459f1cf7d3b7
toomanyobjs==0.0.0 \
--hash=sha256:997e9399a33d4884eb535b6cc2c02705ed9ae1bcff56afbbf3ea81fc4ac9ab95
unpaddedbase32==0.1.0 \ unpaddedbase32==0.1.0 \
--hash=sha256:5e4143fcaf77c9c6b4f60d18301c7570f0dac561dcf9b9aed8b5ba6ead7f218c --hash=sha256:5e4143fcaf77c9c6b4f60d18301c7570f0dac561dcf9b9aed8b5ba6ead7f218c
urllib3==1.24.2 \ urllib3==1.24.2 \