diff --git a/run_tests.sh b/run_tests.sh index 74cb596f..de78749d 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -3,14 +3,22 @@ rm -rf testdata; mkdir testdata; ran=0 -SECONDS=0 ; +SECONDS=0 ; close () { rm -rf testdata; exit 10; } for f in tests/*.py; do - python3 "$f" || close # if needed + python3 "$f" || close # if needed let "ran++" done -echo "ran $ran test files successfully in $SECONDS seconds" +echo "ran $ran unittests. Unittest Time: $SECONDS" +ran=0; + +for f in tests/integration-tests/*.py; do + python3 "$f" || close # if needed + let "ran++" +done +echo "ran $ran integration test tests." +echo "total test time $SECONDS" diff --git a/scripts/disable-dev-config.py b/scripts/disable-dev-config.py index a55566bf..07eba914 100755 --- a/scripts/disable-dev-config.py +++ b/scripts/disable-dev-config.py @@ -17,6 +17,7 @@ conf['general']['random_bind_ip'] = True conf['onboarding']['done'] = False conf['general']['minimum_block_pow'] = 5 conf['general']['minimum_send_pow'] = 5 +conf['log']['file']['remove_on_exit'] = True json.dump(conf, open('static-data/default_config.json', 'w'), sort_keys=True, indent=4) diff --git a/scripts/enable-dev-config.py b/scripts/enable-dev-config.py index 3f89d25f..835620d5 100755 --- a/scripts/enable-dev-config.py +++ b/scripts/enable-dev-config.py @@ -18,6 +18,7 @@ conf['general']['random_bind_ip'] = False conf['onboarding']['done'] = True conf['general']['minimum_block_pow'] = 4 conf['general']['minimum_send_pow'] = 4 +conf['log']['file']['remove_on_exit'] = False json.dump(conf, open('static-data/default_config.json', 'w'), sort_keys=True, indent=4) diff --git a/src/__init__.py b/src/__init__.py index 1681dcfb..8fd207cf 100755 --- a/src/__init__.py +++ b/src/__init__.py @@ -68,6 +68,7 @@ setup.setup_config() import config # noqa from utils import identifyhome # noqa +import filepaths # noqa if config.get('advanced.security_auditing', True): try: @@ -93,7 +94,10 @@ if ran_as_script: # If the setting is there, shred log file on exit if config.get('log.file.remove_on_exit', True): - nuke.clean(config.get_config_file()) + try: + nuke.clean(filepaths.log_file) + except FileNotFoundError: + pass # Cleanup standard out/err because Python refuses to do it itsself try: diff --git a/src/bigbrother/ministry/ofexec.py b/src/bigbrother/ministry/ofexec.py index 51b1b857..9df6460f 100644 --- a/src/bigbrother/ministry/ofexec.py +++ b/src/bigbrother/ministry/ofexec.py @@ -72,7 +72,7 @@ def block_exec(event, info): if info[0].co_filename.endswith(source): return - if home + 'plugins/' in info[0].co_filename: + if 'plugins/' in info[0].co_filename: return logger.warn('POSSIBLE EXPLOIT DETECTED, SEE LOGS', terminal=True) diff --git a/src/etc/onionrvalues.py b/src/etc/onionrvalues.py index 49515836..26f08951 100755 --- a/src/etc/onionrvalues.py +++ b/src/etc/onionrvalues.py @@ -23,10 +23,10 @@ import filepaths DENIABLE_PEER_ADDRESS = "OVPCZLOXD6DC5JHX4EQ3PSOGAZ3T24F75HQLIUZSDSMYPEOXCPFA" PASSWORD_LENGTH = 25 ONIONR_TAGLINE = 'Private P2P Communication - GPLv3 - https://Onionr.net' -ONIONR_VERSION = '2.0.0' +ONIONR_VERSION = '3.0.0' ONIONR_VERSION_CODENAME = 'Genesis' ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION) -API_VERSION = '0' # increments of 1; only change when something fundamental about how the API works changes. This way other nodes know how to communicate without learning too much information about you. +API_VERSION = '1' # increments of 1; only change when something fundamental about how the API works changes. This way other nodes know how to communicate without learning too much information about you. MIN_PY_VERSION = 7 # min version of 7 so we can take advantage of non-cyclic type hints DEVELOPMENT_MODE = False """limit type length for a block (soft enforced, ignored if invalid but block still stored).""" @@ -48,6 +48,8 @@ WSGI_SERVER_REQUEST_TIMEOUT_SECS = 120 MAX_NEW_PEER_QUEUE = 1000 +BLOCK_EXPORT_FILE_EXT = '.dat' + # Begin OnionrValues migrated values """30 days is plenty of time for someone to decide to renew a block""" diff --git a/src/filepaths/__init__.py b/src/filepaths/__init__.py index 55629ec8..04fde83c 100644 --- a/src/filepaths/__init__.py +++ b/src/filepaths/__init__.py @@ -32,3 +32,5 @@ data_nonce_file = home + 'block-nonces.dat' keys_file = home + 'keys.txt' onboarding_mark_file = home + 'onboarding-completed' + +log_file = home + 'onionr.log' diff --git a/src/logger/settings.py b/src/logger/settings.py index 0f02e19b..a1ef11da 100644 --- a/src/logger/settings.py +++ b/src/logger/settings.py @@ -19,6 +19,7 @@ ''' import os from utils import identifyhome +import filepaths data_home = os.environ.get('ONIONR_LOG_DIR', identifyhome.identify_home()) # Use the bitwise operators to merge these settings @@ -39,7 +40,8 @@ MAX_LOG_FILE_LINES = 10000 _type = OUTPUT_TO_CONSOLE | USE_ANSI # the default settings for logging _level = LEVEL_DEBUG # the lowest level to log -_outputfile = '%s/onionr.log' % (data_home,) # the file to log to +# the file to log to +_outputfile = filepaths.log_file def set_settings(type): ''' diff --git a/src/onionrcommands/exportblocks.py b/src/onionrcommands/exportblocks.py index 775537e9..df499fe2 100755 --- a/src/onionrcommands/exportblocks.py +++ b/src/onionrcommands/exportblocks.py @@ -9,6 +9,9 @@ import onionrstorage from utils import createdirs from onionrutils import stringvalidators import filepaths + +import os +from coredb import blockmetadb """ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/onionrsetup/dbcreator.py b/src/onionrsetup/dbcreator.py index ed5ffdd6..d9b8e626 100755 --- a/src/onionrsetup/dbcreator.py +++ b/src/onionrsetup/dbcreator.py @@ -1,9 +1,8 @@ -''' - Onionr - Private P2P Communication +"""Onionr - Private P2P Communication. - DBCreator, creates sqlite3 databases used by Onionr -''' -''' +DBCreator, creates sqlite3 databases used by Onionr +""" +""" This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or @@ -16,7 +15,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -''' +""" import sqlite3, os from coredb import dbfiles import filepaths diff --git a/src/onionrsetup/defaultpluginsetup.py b/src/onionrsetup/defaultpluginsetup.py index 422248f2..48320908 100644 --- a/src/onionrsetup/defaultpluginsetup.py +++ b/src/onionrsetup/defaultpluginsetup.py @@ -22,13 +22,19 @@ import os, shutil import onionrplugins as plugins import logger import filepaths +from utils.readstatic import get_static_dir def setup_default_plugins(): # Copy default plugins into plugins folder if not os.path.exists(plugins.get_plugins_folder()): - if os.path.exists('../static-data/default-plugins/'): - names = [f for f in os.listdir("../static-data/default-plugins/")] - shutil.copytree('../static-data/default-plugins/', plugins.get_plugins_folder()) + if os.path.exists(get_static_dir() + '/default-plugins/'): + names = [f for f in os.listdir(get_static_dir() + '/default-plugins/')] + try: + shutil.copytree( + get_static_dir() + '/default-plugins/', + plugins.get_plugins_folder()) + except FileExistsError: + pass # Enable plugins for name in names: @@ -39,6 +45,8 @@ def setup_default_plugins(): if not os.path.exists(plugins.get_plugin_data_folder(name)): try: os.mkdir(plugins.get_plugin_data_folder(name)) + except FileExistsError: + pass except Exception as e: #logger.warn('Error enabling plugin: ' + str(e), terminal=True) plugins.disable(name, stop_event = False) diff --git a/src/onionrstorage/__init__.py b/src/onionrstorage/__init__.py index 636db538..7e198663 100755 --- a/src/onionrstorage/__init__.py +++ b/src/onionrstorage/__init__.py @@ -77,7 +77,7 @@ def store(data, blockHash=''): raise ValueError('Hash specified does not meet internal hash check') else: blockHash = ourHash - + if DB_ENTRY_SIZE_LIMIT >= sys.getsizeof(data): _dbInsert(blockHash, data) else: @@ -86,21 +86,23 @@ def store(data, blockHash=''): def getData(bHash): + if not stringvalidators.validate_hash(bHash): raise ValueError bHash = bytesconverter.bytes_to_str(bHash) - + bHash = bHash.strip() # First check DB for data entry by hash # if no entry, check disk # If no entry in either, raise an exception retData = None fileLocation = '%s/%s.dat' % (filepaths.block_data_location, bHash) - not_found_msg = "Flock data not found for: " + not_found_msg = "Block data not found for: " if os.path.exists(fileLocation): with open(fileLocation, 'rb') as block: retData = block.read() else: retData = _dbFetch(bHash) + if retData is None: raise onionrexceptions.NoDataAvailable(not_found_msg + str(bHash)) return retData diff --git a/src/onionrstorage/setdata.py b/src/onionrstorage/setdata.py index 5c949b96..cbff6e21 100644 --- a/src/onionrstorage/setdata.py +++ b/src/onionrstorage/setdata.py @@ -1,13 +1,31 @@ -import sys, sqlite3 +"""Onionr - Private P2P Communication. + +Test Onionr as it is running +""" +import sys +import sqlite3 + import onionrstorage, onionrexceptions, onionrcrypto as crypto import filepaths from onionrblocks import storagecounter from coredb import dbfiles from onionrutils import blockmetadata, bytesconverter +""" + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +""" def set_data(data)->str: - ''' - Set the data assciated with a hash - ''' + """Set the data assciated with a hash.""" storage_counter = storagecounter.StorageCounter() data = data dataSize = sys.getsizeof(data) diff --git a/src/runtests/__init__.py b/src/runtests/__init__.py index 9da68437..4299d386 100644 --- a/src/runtests/__init__.py +++ b/src/runtests/__init__.py @@ -11,6 +11,7 @@ from . import uicheck, inserttest, stresstest from . import ownnode from .webpasstest import webpass_test from .osver import test_os_ver_endpoint +from .clearnettor import test_clearnet_tor_request """ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -32,7 +33,8 @@ RUN_TESTS = [uicheck.check_ui, ownnode.test_own_node, stresstest.stress_test_block_insert, webpass_test, - test_os_ver_endpoint + test_os_ver_endpoint, + test_clearnet_tor_request ] SUCCESS_FILE = os.path.dirname(os.path.realpath(__file__)) + '/../../tests/runtime-result.txt' diff --git a/src/runtests/clearnettor.py b/src/runtests/clearnettor.py new file mode 100644 index 00000000..30419f9c --- /dev/null +++ b/src/runtests/clearnettor.py @@ -0,0 +1,61 @@ +"""Onionr - Private P2P Communication. + +Ensure that clearnet cannot be reached +""" +from threading import Thread + +from onionrutils.basicrequests import do_get_request +from onionrutils import localcommand +import logger +import config + +""" + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +""" + + +def test_clearnet_tor_request(testmanager): + """Ensure that Tor cannot request clearnet address. + + Does not run if Tor is being reused + """ + + config.reload() + leak_result = "" + + if config.get('tor.use_existing_tor', False): + logger.warn( + "Can't ensure Tor reqs to clearnet won't happen when reusing Tor") + return + + + socks_port = localcommand.local_command('/gettorsocks') + + # Don't worry, this request isn't meant to go through, + # but if it did it would be through Tor + + try: + leak_result: str = do_get_request( + 'https://onionr.net/404', + port=socks_port, ignoreAPI=True).lower() + except AttributeError: + leak_result = "" + except Exception as e: + logger.warn(str(e)) + try: + if 'not found' in leak_result: + logger.error('Tor was able to request a clearnet site') + raise ValueError('Tor was able to request a clearnet site') + except TypeError: + pass diff --git a/src/runtests/inserttest.py b/src/runtests/inserttest.py index 39bd0e00..63b3a57f 100644 --- a/src/runtests/inserttest.py +++ b/src/runtests/inserttest.py @@ -11,7 +11,7 @@ def _check_remote_node(testmanager): def insert_bin_test(testmanager): data = os.urandom(32) - b_hash = onionrblocks.insert(data, ) + b_hash = onionrblocks.insert(data) if not b_hash in coredb.blockmetadb.get_block_list(): logger.error(str(b_hash) + 'is not in bl') diff --git a/src/runtests/ownnode.py b/src/runtests/ownnode.py index 1c4a3e5f..51a0c76d 100644 --- a/src/runtests/ownnode.py +++ b/src/runtests/ownnode.py @@ -25,14 +25,16 @@ from onionrutils import localcommand def test_own_node(test_manager): + return socks_port = localcommand.local_command('/gettorsocks') if config.get('general.security_level', 0) > 0: return own_tor_address = gettransports.get()[0] if 'this is an onionr node' \ not in basicrequests.do_get_request('http://' + own_tor_address, - port=socks_port, ignoreAPI=True).lower(): - logger.warn('Own node not reachable in test') + port=socks_port, + ignoreAPI=True).lower(): + logger.warn(f'Own node not reachable in test {own_tor_address}') raise ValueError diff --git a/static-data/default-plugins/flow/info.json b/static-data/default-plugins/flow/info.json index d8b98bb5..13e05d76 100755 --- a/static-data/default-plugins/flow/info.json +++ b/static-data/default-plugins/flow/info.json @@ -1,4 +1,4 @@ { "name": "flow", - "version": "0.0.1", + "version": "0.1.0", "author": "onionr" } \ No newline at end of file diff --git a/static-data/default-plugins/flow/main.py b/static-data/default-plugins/flow/main.py index 0d421572..f2d95795 100755 --- a/static-data/default-plugins/flow/main.py +++ b/static-data/default-plugins/flow/main.py @@ -39,8 +39,9 @@ flask_blueprint = flowapi.flask_blueprint security_whitelist = ['staticfiles.boardContent', 'staticfiles.board'] plugin_name = 'flow' -PLUGIN_VERSION = '0.0.1' +PLUGIN_VERSION = '0.1.0' +EXPIRE_TIME = 43200 class OnionrFlow: def __init__(self): @@ -75,7 +76,7 @@ class OnionrFlow: else: if message == "q": self.flowRunning = False - expireTime = epoch.get_epoch() + 43200 + expireTime = epoch.get_epoch() + EXPIRE_TIME if len(message) > 0: logger.info('Inserting message as block...', terminal=True) onionrblocks.insert(message, header='brd', @@ -118,6 +119,24 @@ def on_flow_cmd(api, data=None): OnionrFlow().start() +def on_flowsend_cmd(api, data=None): + err_msg = "Second arg is board name, third is quoted message" + try: + sys.argv[2] + except IndexError: + logger.error(err_msg, terminal=True) + try: + sys.argv[3] + except IndexError: + logger.error(err_msg, terminal=True) + + bl = onionrblocks.insert(sys.argv[3], header='brd', + expire=(EXPIRE_TIME + epoch.get_epoch()), + meta={'ch': sys.argv[2]}) + print(bl) + + + def on_softreset(api, data=None): try: os.remove(identifyhome.identify_home() + '/board-index.cache.json') diff --git a/static-data/default_config.json b/static-data/default_config.json index d4f131fe..7ecbce76 100755 --- a/static-data/default_config.json +++ b/static-data/default_config.json @@ -46,7 +46,9 @@ "minimum_score": -100 }, "plugins": { - "disabled": [], + "disabled": [ + "chat" + ], "enabled": [] }, "timers": { diff --git a/tests/integration-tests/details-test.py b/tests/integration-tests/details-test.py new file mode 100644 index 00000000..f952d83d --- /dev/null +++ b/tests/integration-tests/details-test.py @@ -0,0 +1,18 @@ +import sys +import os +from subprocess import Popen, PIPE +import uuid + +TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/' +os.environ["ONIONR_HOME"] = TEST_DIR + +print(f'running integration test for {__file__}') + +with Popen(['./onionr.sh', 'details'], stdout=PIPE) as onionr_proc: + output = onionr_proc.stdout.read().decode() +if onionr_proc.returncode != 0: + raise ValueError('Raised non zero exit ' + str(onionr_proc.returncode)) + +for word in ['Node', 'Human-readable']: + if word not in output: + raise ValueError(word + " not in " + output) diff --git a/tests/integration-tests/export-test.py b/tests/integration-tests/export-test.py new file mode 100644 index 00000000..fa374f3b --- /dev/null +++ b/tests/integration-tests/export-test.py @@ -0,0 +1,40 @@ +from unittest.mock import patch +import sys, os +sys.path.append(".") +sys.path.append("src/") +import unittest, uuid +TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/' +print("Test directory:", TEST_DIR) +os.environ["ONIONR_HOME"] = TEST_DIR +from utils import createdirs +from onionrcommands import parser +import onionrsetup as setup +from netcontroller.torcontrol import customtorrc +from utils import createdirs +from onionrsetup import setup_config, setup_default_plugins +from coredb import blockmetadb +from etc.onionrvalues import BLOCK_EXPORT_FILE_EXT + +createdirs.create_dirs() +setup_config() +setup_default_plugins() +import config +from filepaths import export_location + +class OnionrTests(unittest.TestCase): + def test_export(self): + testargs = ["onionr.py", "flowsend", "tests", "hello"] + with patch.object(sys, 'argv', testargs): + parser.register() + bl = blockmetadb.get_block_list()[0] + testargs = ["onionr.py", "export-block", bl] + with patch.object(sys, 'argv', testargs): + parser.register() + + with open(export_location + '/' + bl + BLOCK_EXPORT_FILE_EXT, 'rb') as f: + + if b'hello' not in f.read(): + raise ValueError('No exported block') + + +unittest.main() diff --git a/tests/integration-tests/no-command-run.py b/tests/integration-tests/no-command-run.py new file mode 100644 index 00000000..48d051fa --- /dev/null +++ b/tests/integration-tests/no-command-run.py @@ -0,0 +1,16 @@ +import sys +import os +from subprocess import Popen, PIPE +import uuid + +TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/' +os.environ["ONIONR_HOME"] = TEST_DIR + +print(f'running integration test for {__file__}') +with Popen(['./onionr.sh'], stdout=PIPE) as onionr_proc: + output = onionr_proc.stdout.read().decode() +if onionr_proc.returncode != 0: + raise ValueError('Raised non zero exit ' + str(onionr_proc.returncode)) + +if output != '': + raise ValueError('No command run returned non-blank output') diff --git a/tests/runtime-result.txt b/tests/runtime-result.txt index 804d12b3..afed5bb1 100644 --- a/tests/runtime-result.txt +++ b/tests/runtime-result.txt @@ -1 +1 @@ -1580971981 \ No newline at end of file +1581152327 \ No newline at end of file