Merge branch 'node-profiling' of gitlab.com:beardog/Onionr into node-profiling

This commit is contained in:
Kevin Froman 2018-07-31 00:28:41 -05:00
commit 34a970d008
28 changed files with 150 additions and 34 deletions

View File

@ -24,7 +24,7 @@ from gevent.wsgi import WSGIServer
import sys, random, threading, hmac, hashlib, base64, time, math, os, logger, config import sys, random, threading, hmac, hashlib, base64, time, math, os, logger, config
from core import Core from core import Core
from onionrblockapi import Block from onionrblockapi import Block
import onionrutils, onionrcrypto, blockimporter import onionrutils, onionrcrypto, blockimporter, onionrevents as events
class API: class API:
''' '''
@ -94,6 +94,7 @@ class API:
''' '''
Simply define the request as not having yet failed, before every request. Simply define the request as not having yet failed, before every request.
''' '''
self.requestFailed = False self.requestFailed = False
return return
@ -116,20 +117,63 @@ class API:
return resp return resp
@app.route('/client/ui/<path:path>') @app.route('/www/private/<path:path>')
def webUI(path): def www_private(path):
startTime = math.floor(time.time()) startTime = math.floor(time.time())
if request.args.get('timingToken') is None: if request.args.get('timingToken') is None:
timingToken = '' timingToken = ''
else: else:
timingToken = request.args.get('timingToken') timingToken = request.args.get('timingToken')
if not config.get("www.private.run", True):
abort(403)
self.validateHost('private') self.validateHost('private')
endTime = math.floor(time.time()) endTime = math.floor(time.time())
elapsed = endTime - startTime elapsed = endTime - startTime
if not hmac.compare_digest(timingToken, self.timeBypassToken): if not hmac.compare_digest(timingToken, self.timeBypassToken):
if elapsed < self._privateDelayTime: if elapsed < self._privateDelayTime:
time.sleep(self._privateDelayTime - elapsed) time.sleep(self._privateDelayTime - elapsed)
return send_from_directory('static-data/ui/dist/', path)
return send_from_directory('static-data/www/private/', path)
@app.route('/www/public/<path:path>')
def www_public(path):
if not config.get("www.public.run", True):
abort(403)
self.validateHost('public')
return send_from_directory('static-data/www/public/', path)
@app.route('/ui/<path:path>')
def ui_private(path):
startTime = math.floor(time.time())
if request.args.get('timingToken') is None:
timingToken = ''
else:
timingToken = request.args.get('timingToken')
if not config.get("www.ui.run", True):
abort(403)
if config.get("www.ui.private", True):
self.validateHost('private')
else:
self.validateHost('public')
endTime = math.floor(time.time())
elapsed = endTime - startTime
if not hmac.compare_digest(timingToken, self.timeBypassToken):
if elapsed < self._privateDelayTime:
time.sleep(self._privateDelayTime - elapsed)
return send_from_directory('static-data/www/ui/dist/', path)
@app.route('/client/') @app.route('/client/')
def private_handler(): def private_handler():
@ -150,6 +194,9 @@ class API:
if not self.validateToken(token): if not self.validateToken(token):
abort(403) abort(403)
events.event('webapi_private', onionr = None, data = {'action' : action, 'data' : data, 'timingToken' : timingToken, 'token' : token})
self.validateHost('private') self.validateHost('private')
if action == 'hello': if action == 'hello':
resp = Response('Hello, World! ' + request.host) resp = Response('Hello, World! ' + request.host)
@ -198,12 +245,12 @@ class API:
response['hash'] = hash response['hash'] = hash
response['reason'] = 'Successfully wrote block to file' response['reason'] = 'Successfully wrote block to file'
else: else:
response['reason'] = 'Faield to save the block' response['reason'] = 'Failed to save the block'
except Exception as e: except Exception as e:
logger.debug('insertBlock api request failed', error = e) logger.debug('insertBlock api request failed', error = e)
resp = Response(json.dumps(response)) resp = Response(json.dumps(response))
elif action in callbacks['private']: elif action in API.callbacks['private']:
resp = Response(str(getCallback(action, scope = 'private')(request))) resp = Response(str(getCallback(action, scope = 'private')(request)))
else: else:
resp = Response('(O_o) Dude what? (invalid command)') resp = Response('(O_o) Dude what? (invalid command)')
@ -257,6 +304,9 @@ class API:
data = data data = data
except: except:
data = '' data = ''
events.event('webapi_public', onionr = None, data = {'action' : action, 'data' : data, 'requestingPeer' : requestingPeer, 'request' : request})
if action == 'firstConnect': if action == 'firstConnect':
pass pass
elif action == 'ping': elif action == 'ping':
@ -299,7 +349,7 @@ class API:
peers = self._core.listPeers(getPow=True) peers = self._core.listPeers(getPow=True)
response = ','.join(peers) response = ','.join(peers)
resp = Response(response) resp = Response(response)
elif action in callbacks['public']: elif action in API.callbacks['public']:
resp = Response(str(getCallback(action, scope = 'public')(request))) resp = Response(str(getCallback(action, scope = 'public')(request)))
else: else:
resp = Response("") resp = Response("")
@ -373,29 +423,29 @@ class API:
sys.exit(1) sys.exit(1)
def setCallback(action, callback, scope = 'public'): def setCallback(action, callback, scope = 'public'):
if not scope in callbacks: if not scope in API.callbacks:
return False return False
callbacks[scope][action] = callback API.callbacks[scope][action] = callback
return True return True
def removeCallback(action, scope = 'public'): def removeCallback(action, scope = 'public'):
if (not scope in callbacks) or (not action in callbacks[scope]): if (not scope in API.callbacks) or (not action in API.callbacks[scope]):
return False return False
del callbacks[scope][action] del API.callbacks[scope][action]
return True return True
def getCallback(action, scope = 'public'): def getCallback(action, scope = 'public'):
if (not scope in callbacks) or (not action in callbacks[scope]): if (not scope in API.callbacks) or (not action in API.callbacks[scope]):
return None return None
return callbacks[scope][action] return API.callbacks[scope][action]
def getCallbacks(scope = None): def getCallbacks(scope = None):
if (not scope is None) and (scope in callbacks): if (not scope is None) and (scope in API.callbacks):
return callbacks[scope] return API.callbacks[scope]
return callbacks return API.callbacks

View File

@ -19,8 +19,8 @@
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
import sys, os, core, config, json, onionrblockapi as block, requests, time, logger, threading, onionrplugins as plugins, base64, onionr import sys, os, core, config, json, requests, time, logger, threading, base64, onionr
import onionrexceptions, onionrpeers import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block
from defusedxml import minidom from defusedxml import minidom
class OnionrCommunicatorDaemon: class OnionrCommunicatorDaemon:
@ -363,6 +363,8 @@ class OnionrCommunicatorDaemon:
cmd = self._core.daemonQueue() cmd = self._core.daemonQueue()
if cmd is not False: if cmd is not False:
events.event('daemon_command', onionr = None, data = {'cmd' : cmd})
if cmd[0] == 'shutdown': if cmd[0] == 'shutdown':
self.shutdown = True self.shutdown = True
elif cmd[0] == 'announceNode': elif cmd[0] == 'announceNode':
@ -381,6 +383,7 @@ class OnionrCommunicatorDaemon:
threading.Thread(target=self.uploadBlock).start() threading.Thread(target=self.uploadBlock).start()
else: else:
logger.info('Recieved daemonQueue command:' + cmd[0]) logger.info('Recieved daemonQueue command:' + cmd[0])
self.decrementThreadCount('daemonCommands') self.decrementThreadCount('daemonCommands')
def uploadBlock(self): def uploadBlock(self):
@ -427,6 +430,7 @@ class OnionrCommunicatorDaemon:
time.sleep(1) time.sleep(1)
else: else:
# This executes if the api is NOT detected to be running # This executes if the api is NOT detected to be running
events.event('daemon_crash', onionr = None, data = {})
logger.error('Daemon detected API crash (or otherwise unable to reach API after long time), stopping...') logger.error('Daemon detected API crash (or otherwise unable to reach API after long time), stopping...')
self.shutdown = True self.shutdown = True
self.decrementThreadCount('detectAPICrash') self.decrementThreadCount('detectAPICrash')

View File

@ -100,7 +100,7 @@ DataDirectory data/tordata/
logger.fatal('Failed to start Tor. Try killing any other Tor processes owned by this user.') logger.fatal('Failed to start Tor. Try killing any other Tor processes owned by this user.')
return False return False
except KeyboardInterrupt: except KeyboardInterrupt:
logger.fatal("Got keyboard interrupt") logger.fatal("Got keyboard interrupt.")
return False return False
logger.debug('Finished starting Tor.', timestamp=True) logger.debug('Finished starting Tor.', timestamp=True)

View File

@ -40,9 +40,9 @@ except ImportError:
raise Exception("You need the PySocks module (for use with socks5 proxy to use Tor)") raise Exception("You need the PySocks module (for use with socks5 proxy to use Tor)")
ONIONR_TAGLINE = 'Anonymous P2P Platform - GPLv3 - https://Onionr.VoidNet.Tech' ONIONR_TAGLINE = 'Anonymous P2P Platform - GPLv3 - https://Onionr.VoidNet.Tech'
ONIONR_VERSION = '0.1.0' # for debugging and stuff ONIONR_VERSION = '0.1.1' # for debugging and stuff
ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION) ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION)
API_VERSION = '4' # increments of 1; only change when something fundemental about how the API works changes. This way other nodes knows how to communicate without learning too much information about you. API_VERSION = '4' # increments of 1; only change when something fundemental about how the API works changes. This way other nodes know how to communicate without learning too much information about you.
class Onionr: class Onionr:
def __init__(self): def __init__(self):

View File

@ -33,10 +33,10 @@ def __event_caller(event_name, data = {}, onionr = None):
try: try:
call(plugins.get_plugin(plugin), event_name, data, get_pluginapi(onionr, data)) call(plugins.get_plugin(plugin), event_name, data, get_pluginapi(onionr, data))
except ModuleNotFoundError as e: except ModuleNotFoundError as e:
logger.warn('Disabling nonexistant plugin \"' + plugin + '\"...') logger.warn('Disabling nonexistant plugin "%s"...' % plugin)
plugins.disable(plugin, onionr, stop_event = False) plugins.disable(plugin, onionr, stop_event = False)
except Exception as e: except Exception as e:
logger.warn('Event \"' + event_name + '\" failed for plugin \"' + plugin + '\".') logger.warn('Event "%s" failed for plugin "%s".' % (event_name, plugin))
logger.debug(str(e)) logger.debug(str(e))

View File

@ -49,5 +49,6 @@ class InvalidProof(Exception):
# network level exceptions # network level exceptions
class MissingPort(Exception): class MissingPort(Exception):
pass pass
class InvalidAddress(Exception): class InvalidAddress(Exception):
pass pass

View File

@ -199,7 +199,7 @@ class OnionrUtils:
def getBlockMetadataFromData(self, blockData): def getBlockMetadataFromData(self, blockData):
''' '''
accepts block contents as string, returns a tuple of metadata, meta (meta being internal metadata, which will be returned as an encrypted base64 string if it is encrypted, dict if not). accepts block contents as string, returns a tuple of metadata, meta (meta being internal metadata, which will be returned as an encrypted base64 string if it is encrypted, dict if not).
''' '''
meta = {} meta = {}
metadata = {} metadata = {}
@ -208,7 +208,7 @@ class OnionrUtils:
blockData = blockData.encode() blockData = blockData.encode()
except AttributeError: except AttributeError:
pass pass
try: try:
metadata = json.loads(blockData[:blockData.find(b'\n')].decode()) metadata = json.loads(blockData[:blockData.find(b'\n')].decode())
except json.decoder.JSONDecodeError: except json.decoder.JSONDecodeError:
@ -221,7 +221,7 @@ class OnionrUtils:
meta = json.loads(metadata['meta']) meta = json.loads(metadata['meta'])
except KeyError: except KeyError:
pass pass
meta = metadata['meta'] meta = metadata['meta']
return (metadata, meta, data) return (metadata, meta, data)
def checkPort(self, port, host=''): def checkPort(self, port, host=''):
@ -251,7 +251,7 @@ class OnionrUtils:
return False return False
else: else:
return True return True
def processBlockMetadata(self, blockHash): def processBlockMetadata(self, blockHash):
''' '''
Read metadata from a block and cache it to the block database Read metadata from a block and cache it to the block database
@ -269,7 +269,7 @@ class OnionrUtils:
def escapeAnsi(self, line): def escapeAnsi(self, line):
''' '''
Remove ANSI escape codes from a string with regex Remove ANSI escape codes from a string with regex
taken or adapted from: https://stackoverflow.com/a/38662876 taken or adapted from: https://stackoverflow.com/a/38662876
''' '''
ansi_escape = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -/]*[@-~]') ansi_escape = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -/]*[@-~]')
@ -331,12 +331,12 @@ class OnionrUtils:
retVal = False retVal = False
return retVal return retVal
def validateMetadata(self, metadata): def validateMetadata(self, metadata):
'''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string''' '''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string'''
# TODO, make this check sane sizes # TODO, make this check sane sizes
retData = False retData = False
# convert to dict if it is json string # convert to dict if it is json string
if type(metadata) is str: if type(metadata) is str:
try: try:
@ -382,7 +382,7 @@ class OnionrUtils:
else: else:
retVal = True retVal = True
return retVal return retVal
def isIntegerString(self, data): def isIntegerString(self, data):
'''Check if a string is a valid base10 integer''' '''Check if a string is a valid base10 integer'''
try: try:

View File

@ -2,8 +2,26 @@
"general" : { "general" : {
"dev_mode": true, "dev_mode": true,
"display_header" : true, "display_header" : true,
"dc_response": true,
"dc_execcallbacks" : true "direct_connect" : {
"respond" : true,
"execute_callbacks" : true
}
},
"www" : {
"public" : {
"run" : true
},
"private" : {
"run" : true
},
"ui" : {
"run" : true,
"private" : true
}
}, },
"client" : { "client" : {

View File

@ -1 +0,0 @@
Static files for Onionr's web ui, change at your own risk.

View File

@ -0,0 +1,44 @@
# Onionr UI
## About
The default GUI for Onionr
## Setup
To compile the application, simply execute the following:
```
python3 compile.py
```
If you are wanting to compile Onionr UI for another language, execute the following, replacing `[lang]` with the target language (supported languages include `eng` for English, `spa` para español, and `zho`为中国人):
```
python3 compile.py [lang]
```
## FAQ
### Why "compile" anyway?
This web application is compiled for a few reasons:
1. To make it easier to update; this way, we do not have to update the header in every file if we want to change something about it.
2. To make the application smaller in size; there is less duplicated code when the code like the header and footer can be stored in an individual file rather than every file.
3. For multi-language support; with the Python "tags" feature, we can reference strings by variable name, and based on a language file, they can be dynamically inserted into the page on compilation.
4. For compile-time customizations.
### What exactly happens when you compile?
Upon compilation, files from the `src/` directory will be copied to `dist/` directory, header and footers will be injected in the proper places, and Python "tags" will be interpreted.
### How do Python "tags" work?
There are two types of Python "tags":
1. Logic tags (`<$ logic $>`): These tags allow you to perform logic at compile time. Example: `<$ import datetime; lastUpdate = datetime.datetime.now() $>`: This gets the current time while compiling, then stores it in `lastUpdate`.
2. Data tags (`<$= data $>`): These tags take whatever the return value of the statement in the tags is, and write it directly to the page. Example: `<$= 'This application was compiled at %s.' % lastUpdate $>`: This will write the message in the string in the tags to the page.
**Note:** Logic tags take a higher priority and will always be interpreted first.
### How does the language feature work?
When you use a data tag to write a string to the page (e.g. `<$= LANG.HELLO_WORLD $>`), the language feature simply takes dictionary of the language that is currently being used from the language map file (`lang.json`), then searches for the key (being the variable name after the characters `LANG.` in the data tag, like `HELLO_WORLD` from the example before). It then writes that string to the page. Language variables are always prefixed with `LANG.` and should always be uppercase (as they are a constant).
### I changed a few things in the application and tried to view the updates in my browser, but nothing changed!
You most likely forgot to compile. Try running `python3 compile.py` and check again. If you are still having issues, [open up an issue](https://gitlab.com/beardog/Onionr/issues/new?issue[title]=Onionr UI not updating after compiling).

View File

Before

Width:  |  Height:  |  Size: 6.6 KiB

After

Width:  |  Height:  |  Size: 6.6 KiB

View File

Before

Width:  |  Height:  |  Size: 6.6 KiB

After

Width:  |  Height:  |  Size: 6.6 KiB