Merge branch 'node-profiling' of gitlab.com:beardog/Onionr into node-profiling

onionr/api.py

@@ -24,7 +24,7 @@ from gevent.wsgi import WSGIServer
 import sys, random, threading, hmac, hashlib, base64, time, math, os, logger, config
 from core import Core
 from onionrblockapi import Block
-import onionrutils, onionrcrypto, blockimporter
+import onionrutils, onionrcrypto, blockimporter, onionrevents as events
 
 class API:
     '''
@@ -94,6 +94,7 @@ class API:
             '''
                 Simply define the request as not having yet failed, before every request.
             '''
+
             self.requestFailed = False
 
             return
@@ -116,20 +117,63 @@ class API:
 
             return resp
 
-        @app.route('/client/ui/<path:path>')
-        def webUI(path):
+        @app.route('/www/private/<path:path>')
+        def www_private(path):
             startTime = math.floor(time.time())
+
             if request.args.get('timingToken') is None:
                 timingToken = ''
             else:
                 timingToken = request.args.get('timingToken')
+
+            if not config.get("www.private.run", True):
+                abort(403)
+
             self.validateHost('private')
+
             endTime = math.floor(time.time())
             elapsed = endTime - startTime
+
             if not hmac.compare_digest(timingToken, self.timeBypassToken):
                 if elapsed < self._privateDelayTime:
                     time.sleep(self._privateDelayTime - elapsed)
-            return send_from_directory('static-data/ui/dist/', path)
+
+            return send_from_directory('static-data/www/private/', path)
+
+        @app.route('/www/public/<path:path>')
+        def www_public(path):
+            if not config.get("www.public.run", True):
+                abort(403)
+
+            self.validateHost('public')
+
+            return send_from_directory('static-data/www/public/', path)
+
+        @app.route('/ui/<path:path>')
+        def ui_private(path):
+            startTime = math.floor(time.time())
+
+            if request.args.get('timingToken') is None:
+                timingToken = ''
+            else:
+                timingToken = request.args.get('timingToken')
+
+            if not config.get("www.ui.run", True):
+                abort(403)
+
+            if config.get("www.ui.private", True):
+                self.validateHost('private')
+            else:
+                self.validateHost('public')
+
+            endTime = math.floor(time.time())
+            elapsed = endTime - startTime
+
+            if not hmac.compare_digest(timingToken, self.timeBypassToken):
+                if elapsed < self._privateDelayTime:
+                    time.sleep(self._privateDelayTime - elapsed)
+
+            return send_from_directory('static-data/www/ui/dist/', path)
 
         @app.route('/client/')
         def private_handler():
@@ -150,6 +194,9 @@ class API:
 
             if not self.validateToken(token):
                 abort(403)
+
+            events.event('webapi_private', onionr = None, data = {'action' : action, 'data' : data, 'timingToken' : timingToken, 'token' : token})
+
             self.validateHost('private')
             if action == 'hello':
                 resp = Response('Hello, World! ' + request.host)
@@ -198,12 +245,12 @@ class API:
                         response['hash'] = hash
                         response['reason'] = 'Successfully wrote block to file'
                     else:
-                        response['reason'] = 'Faield to save the block'
+                        response['reason'] = 'Failed to save the block'
                 except Exception as e:
                     logger.debug('insertBlock api request failed', error = e)
 
                 resp = Response(json.dumps(response))
-            elif action in callbacks['private']:
+            elif action in API.callbacks['private']:
                 resp = Response(str(getCallback(action, scope = 'private')(request)))
             else:
                 resp = Response('(O_o) Dude what? (invalid command)')
@@ -257,6 +304,9 @@ class API:
                 data = data
             except:
                 data = ''
+
+            events.event('webapi_public', onionr = None, data = {'action' : action, 'data' : data, 'requestingPeer' : requestingPeer, 'request' : request})
+
             if action == 'firstConnect':
                 pass
             elif action == 'ping':
@@ -299,7 +349,7 @@ class API:
                 peers = self._core.listPeers(getPow=True)
                 response = ','.join(peers)
                 resp = Response(response)
-            elif action in callbacks['public']:
+            elif action in API.callbacks['public']:
                 resp = Response(str(getCallback(action, scope = 'public')(request)))
             else:
                 resp = Response("")
@@ -373,29 +423,29 @@ class API:
                 sys.exit(1)
 
     def setCallback(action, callback, scope = 'public'):
-        if not scope in callbacks:
+        if not scope in API.callbacks:
             return False
 
-        callbacks[scope][action] = callback
+        API.callbacks[scope][action] = callback
 
         return True
 
     def removeCallback(action, scope = 'public'):
-        if (not scope in callbacks) or (not action in callbacks[scope]):
+        if (not scope in API.callbacks) or (not action in API.callbacks[scope]):
             return False
 
-        del callbacks[scope][action]
+        del API.callbacks[scope][action]
 
         return True
 
     def getCallback(action, scope = 'public'):
-        if (not scope in callbacks) or (not action in callbacks[scope]):
+        if (not scope in API.callbacks) or (not action in API.callbacks[scope]):
             return None
 
-        return callbacks[scope][action]
+        return API.callbacks[scope][action]
 
     def getCallbacks(scope = None):
-        if (not scope is None) and (scope in callbacks):
-            return callbacks[scope]
+        if (not scope is None) and (scope in API.callbacks):
+            return API.callbacks[scope]
 
-        return callbacks
+        return API.callbacks

onionr/communicator2.py

@@ -19,8 +19,8 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import sys, os, core, config, json, onionrblockapi as block, requests, time, logger, threading, onionrplugins as plugins, base64, onionr
-import onionrexceptions, onionrpeers
+import sys, os, core, config, json, requests, time, logger, threading, base64, onionr
+import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block
 from defusedxml import minidom
 
 class OnionrCommunicatorDaemon:
@@ -363,6 +363,8 @@ class OnionrCommunicatorDaemon:
         cmd = self._core.daemonQueue()
 
         if cmd is not False:
+            events.event('daemon_command', onionr = None, data = {'cmd' : cmd})
+
             if cmd[0] == 'shutdown':
                 self.shutdown = True
             elif cmd[0] == 'announceNode':
@@ -381,6 +383,7 @@ class OnionrCommunicatorDaemon:
                 threading.Thread(target=self.uploadBlock).start()
             else:
                 logger.info('Recieved daemonQueue command:' + cmd[0])
+
         self.decrementThreadCount('daemonCommands')
 
     def uploadBlock(self):
@@ -427,6 +430,7 @@ class OnionrCommunicatorDaemon:
                 time.sleep(1)
             else:
                 # This executes if the api is NOT detected to be running
+                events.event('daemon_crash', onionr = None, data = {})
                 logger.error('Daemon detected API crash (or otherwise unable to reach API after long time), stopping...')
                 self.shutdown = True
         self.decrementThreadCount('detectAPICrash')

onionr/netcontroller.py

@@ -100,7 +100,7 @@ DataDirectory data/tordata/
                 logger.fatal('Failed to start Tor. Try killing any other Tor processes owned by this user.')
                 return False
         except KeyboardInterrupt:
-            logger.fatal("Got keyboard interrupt")
+            logger.fatal("Got keyboard interrupt.")
             return False
 
         logger.debug('Finished starting Tor.', timestamp=True)

onionr/onionr.py

@@ -40,9 +40,9 @@ except ImportError:
     raise Exception("You need the PySocks module (for use with socks5 proxy to use Tor)")
 
 ONIONR_TAGLINE = 'Anonymous P2P Platform - GPLv3 - https://Onionr.VoidNet.Tech'
-ONIONR_VERSION = '0.1.0' # for debugging and stuff
+ONIONR_VERSION = '0.1.1' # for debugging and stuff
 ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION)
-API_VERSION = '4' # increments of 1; only change when something fundemental about how the API works changes. This way other nodes knows how to communicate without learning too much information about you.
+API_VERSION = '4' # increments of 1; only change when something fundemental about how the API works changes. This way other nodes know how to communicate without learning too much information about you.
 
 class Onionr:
     def __init__(self):

onionr/onionrevents.py

@@ -33,10 +33,10 @@ def __event_caller(event_name, data = {}, onionr = None):
         try:
             call(plugins.get_plugin(plugin), event_name, data, get_pluginapi(onionr, data))
         except ModuleNotFoundError as e:
-            logger.warn('Disabling nonexistant plugin \"' + plugin + '\"...')
+            logger.warn('Disabling nonexistant plugin "%s"...' % plugin)
             plugins.disable(plugin, onionr, stop_event = False)
         except Exception as e:
-            logger.warn('Event \"' + event_name + '\" failed for plugin \"' + plugin + '\".')
+            logger.warn('Event "%s" failed for plugin "%s".' % (event_name, plugin))
             logger.debug(str(e))
 
 

onionr/onionrexceptions.py

@@ -49,5 +49,6 @@ class InvalidProof(Exception):
 # network level exceptions
 class MissingPort(Exception):
     pass
+
 class InvalidAddress(Exception):
     pass

onionr/onionrutils.py

@@ -199,7 +199,7 @@ class OnionrUtils:
     def getBlockMetadataFromData(self, blockData):
         '''
             accepts block contents as string, returns a tuple of metadata, meta (meta being internal metadata, which will be returned as an encrypted base64 string if it is encrypted, dict if not).
-            
+
         '''
         meta = {}
         metadata = {}
@@ -208,7 +208,7 @@ class OnionrUtils:
             blockData = blockData.encode()
         except AttributeError:
             pass
-        
+
         try:
             metadata = json.loads(blockData[:blockData.find(b'\n')].decode())
         except json.decoder.JSONDecodeError:
@@ -221,7 +221,7 @@ class OnionrUtils:
                     meta = json.loads(metadata['meta'])
                 except KeyError:
                     pass
-            meta = metadata['meta']       
+            meta = metadata['meta']
         return (metadata, meta, data)
 
     def checkPort(self, port, host=''):
@@ -251,7 +251,7 @@ class OnionrUtils:
             return False
         else:
             return True
-    
+
     def processBlockMetadata(self, blockHash):
         '''
             Read metadata from a block and cache it to the block database
@@ -269,7 +269,7 @@ class OnionrUtils:
     def escapeAnsi(self, line):
         '''
             Remove ANSI escape codes from a string with regex
-            
+
             taken or adapted from: https://stackoverflow.com/a/38662876
         '''
         ansi_escape = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -/]*[@-~]')
@@ -331,12 +331,12 @@ class OnionrUtils:
                 retVal = False
 
         return retVal
-    
+
     def validateMetadata(self, metadata):
         '''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string'''
         # TODO, make this check sane sizes
         retData = False
-        
+
         # convert to dict if it is json string
         if type(metadata) is str:
             try:
@@ -382,7 +382,7 @@ class OnionrUtils:
         else:
             retVal = True
         return retVal
-    
+
     def isIntegerString(self, data):
         '''Check if a string is a valid base10 integer'''
         try:

onionr/static-data/default_config.json

@@ -2,8 +2,26 @@
     "general" : {
         "dev_mode": true,
         "display_header" : true,
-        "dc_response": true,
-        "dc_execcallbacks" : true
+
+        "direct_connect" : {
+            "respond" : true,
+            "execute_callbacks" : true
+        }
+    },
+
+    "www" : {
+        "public" : {
+            "run" : true
+        },
+
+        "private" : {
+            "run" : true
+        },
+
+        "ui" : {
+            "run" : true,
+            "private" : true
+        }
     },
 
     "client" : {

onionr/static-data/ui/README.md

@@ -1 +0,0 @@
-Static files for Onionr's web ui, change at your own risk.

onionr/static-data/www/ui/README.md

@@ -0,0 +1,44 @@
+# Onionr UI
+
+## About
+
+The default GUI for Onionr
+
+## Setup
+
+To compile the application, simply execute the following:
+
+```
+python3 compile.py
+```
+
+If you are wanting to compile Onionr UI for another language, execute the following, replacing `[lang]` with the target language (supported languages include `eng` for English, `spa` para español, and `zho`为中国人):
+
+```
+python3 compile.py [lang]
+```
+
+## FAQ
+### Why "compile" anyway?
+This web application is compiled for a few reasons:
+1. To make it easier to update; this way, we do not have to update the header in every file if we want to change something about it.
+2. To make the application smaller in size; there is less duplicated code when the code like the header and footer can be stored in an individual file rather than every file.
+3. For multi-language support; with the Python "tags" feature, we can reference strings by variable name, and based on a language file, they can be dynamically inserted into the page on compilation.
+4. For compile-time customizations.
+
+### What exactly happens when you compile?
+Upon compilation, files from the `src/` directory will be copied to `dist/` directory, header and footers will be injected in the proper places, and Python "tags" will be interpreted.
+
+
+### How do Python "tags" work?
+There are two types of Python "tags":
+1. Logic tags (`<$ logic $>`): These tags allow you to perform logic at compile time. Example: `<$ import datetime; lastUpdate = datetime.datetime.now() $>`: This gets the current time while compiling, then stores it in `lastUpdate`.
+2. Data tags (`<$= data $>`): These tags take whatever the return value of the statement in the tags is, and write it directly to the page. Example: `<$= 'This application was compiled at %s.' % lastUpdate $>`: This will write the message in the string in the tags to the page.
+
+**Note:** Logic tags take a higher priority and will always be interpreted first.
+
+### How does the language feature work?
+When you use a data tag to write a string to the page (e.g. `<$= LANG.HELLO_WORLD $>`), the language feature simply takes dictionary of the language that is currently being used from the language map file (`lang.json`), then searches for the key (being the variable name after the characters `LANG.` in the data tag, like `HELLO_WORLD` from the example before). It then writes that string to the page. Language variables are always prefixed with `LANG.` and should always be uppercase (as they are a constant).
+
+### I changed a few things in the application and tried to view the updates in my browser, but nothing changed!
+You most likely forgot to compile. Try running `python3 compile.py` and check again. If you are still having issues, [open up an issue](https://gitlab.com/beardog/Onionr/issues/new?issue[title]=Onionr UI not updating after compiling).