bumped network version and main version, lots of test work and some stability improvements

This commit is contained in:
Kevin Froman 2020-02-08 03:07:07 -06:00
parent e77d422fc2
commit 572e29f5d5
24 changed files with 243 additions and 33 deletions

View File

@ -3,14 +3,22 @@ rm -rf testdata;
mkdir testdata; mkdir testdata;
ran=0 ran=0
SECONDS=0 ; SECONDS=0 ;
close () { close () {
rm -rf testdata; rm -rf testdata;
exit 10; exit 10;
} }
for f in tests/*.py; do for f in tests/*.py; do
python3 "$f" || close # if needed python3 "$f" || close # if needed
let "ran++" let "ran++"
done done
echo "ran $ran test files successfully in $SECONDS seconds" echo "ran $ran unittests. Unittest Time: $SECONDS"
ran=0;
for f in tests/integration-tests/*.py; do
python3 "$f" || close # if needed
let "ran++"
done
echo "ran $ran integration test tests."
echo "total test time $SECONDS"

View File

@ -17,6 +17,7 @@ conf['general']['random_bind_ip'] = True
conf['onboarding']['done'] = False conf['onboarding']['done'] = False
conf['general']['minimum_block_pow'] = 5 conf['general']['minimum_block_pow'] = 5
conf['general']['minimum_send_pow'] = 5 conf['general']['minimum_send_pow'] = 5
conf['log']['file']['remove_on_exit'] = True
json.dump(conf, open('static-data/default_config.json', 'w'), sort_keys=True, indent=4) json.dump(conf, open('static-data/default_config.json', 'w'), sort_keys=True, indent=4)

View File

@ -18,6 +18,7 @@ conf['general']['random_bind_ip'] = False
conf['onboarding']['done'] = True conf['onboarding']['done'] = True
conf['general']['minimum_block_pow'] = 4 conf['general']['minimum_block_pow'] = 4
conf['general']['minimum_send_pow'] = 4 conf['general']['minimum_send_pow'] = 4
conf['log']['file']['remove_on_exit'] = False
json.dump(conf, open('static-data/default_config.json', 'w'), sort_keys=True, indent=4) json.dump(conf, open('static-data/default_config.json', 'w'), sort_keys=True, indent=4)

View File

@ -68,6 +68,7 @@ setup.setup_config()
import config # noqa import config # noqa
from utils import identifyhome # noqa from utils import identifyhome # noqa
import filepaths # noqa
if config.get('advanced.security_auditing', True): if config.get('advanced.security_auditing', True):
try: try:
@ -93,7 +94,10 @@ if ran_as_script:
# If the setting is there, shred log file on exit # If the setting is there, shred log file on exit
if config.get('log.file.remove_on_exit', True): if config.get('log.file.remove_on_exit', True):
nuke.clean(config.get_config_file()) try:
nuke.clean(filepaths.log_file)
except FileNotFoundError:
pass
# Cleanup standard out/err because Python refuses to do it itsself # Cleanup standard out/err because Python refuses to do it itsself
try: try:

View File

@ -72,7 +72,7 @@ def block_exec(event, info):
if info[0].co_filename.endswith(source): if info[0].co_filename.endswith(source):
return return
if home + 'plugins/' in info[0].co_filename: if 'plugins/' in info[0].co_filename:
return return
logger.warn('POSSIBLE EXPLOIT DETECTED, SEE LOGS', terminal=True) logger.warn('POSSIBLE EXPLOIT DETECTED, SEE LOGS', terminal=True)

View File

@ -23,10 +23,10 @@ import filepaths
DENIABLE_PEER_ADDRESS = "OVPCZLOXD6DC5JHX4EQ3PSOGAZ3T24F75HQLIUZSDSMYPEOXCPFA" DENIABLE_PEER_ADDRESS = "OVPCZLOXD6DC5JHX4EQ3PSOGAZ3T24F75HQLIUZSDSMYPEOXCPFA"
PASSWORD_LENGTH = 25 PASSWORD_LENGTH = 25
ONIONR_TAGLINE = 'Private P2P Communication - GPLv3 - https://Onionr.net' ONIONR_TAGLINE = 'Private P2P Communication - GPLv3 - https://Onionr.net'
ONIONR_VERSION = '2.0.0' ONIONR_VERSION = '3.0.0'
ONIONR_VERSION_CODENAME = 'Genesis' ONIONR_VERSION_CODENAME = 'Genesis'
ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION) ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION)
API_VERSION = '0' # increments of 1; only change when something fundamental about how the API works changes. This way other nodes know how to communicate without learning too much information about you. API_VERSION = '1' # increments of 1; only change when something fundamental about how the API works changes. This way other nodes know how to communicate without learning too much information about you.
MIN_PY_VERSION = 7 # min version of 7 so we can take advantage of non-cyclic type hints MIN_PY_VERSION = 7 # min version of 7 so we can take advantage of non-cyclic type hints
DEVELOPMENT_MODE = False DEVELOPMENT_MODE = False
"""limit type length for a block (soft enforced, ignored if invalid but block still stored).""" """limit type length for a block (soft enforced, ignored if invalid but block still stored)."""
@ -48,6 +48,8 @@ WSGI_SERVER_REQUEST_TIMEOUT_SECS = 120
MAX_NEW_PEER_QUEUE = 1000 MAX_NEW_PEER_QUEUE = 1000
BLOCK_EXPORT_FILE_EXT = '.dat'
# Begin OnionrValues migrated values # Begin OnionrValues migrated values
"""30 days is plenty of time for someone to decide to renew a block""" """30 days is plenty of time for someone to decide to renew a block"""

View File

@ -32,3 +32,5 @@ data_nonce_file = home + 'block-nonces.dat'
keys_file = home + 'keys.txt' keys_file = home + 'keys.txt'
onboarding_mark_file = home + 'onboarding-completed' onboarding_mark_file = home + 'onboarding-completed'
log_file = home + 'onionr.log'

View File

@ -19,6 +19,7 @@
''' '''
import os import os
from utils import identifyhome from utils import identifyhome
import filepaths
data_home = os.environ.get('ONIONR_LOG_DIR', identifyhome.identify_home()) data_home = os.environ.get('ONIONR_LOG_DIR', identifyhome.identify_home())
# Use the bitwise operators to merge these settings # Use the bitwise operators to merge these settings
@ -39,7 +40,8 @@ MAX_LOG_FILE_LINES = 10000
_type = OUTPUT_TO_CONSOLE | USE_ANSI # the default settings for logging _type = OUTPUT_TO_CONSOLE | USE_ANSI # the default settings for logging
_level = LEVEL_DEBUG # the lowest level to log _level = LEVEL_DEBUG # the lowest level to log
_outputfile = '%s/onionr.log' % (data_home,) # the file to log to # the file to log to
_outputfile = filepaths.log_file
def set_settings(type): def set_settings(type):
''' '''

View File

@ -9,6 +9,9 @@ import onionrstorage
from utils import createdirs from utils import createdirs
from onionrutils import stringvalidators from onionrutils import stringvalidators
import filepaths import filepaths
import os
from coredb import blockmetadb
""" """
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by

View File

@ -1,9 +1,8 @@
''' """Onionr - Private P2P Communication.
Onionr - Private P2P Communication
DBCreator, creates sqlite3 databases used by Onionr DBCreator, creates sqlite3 databases used by Onionr
''' """
''' """
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or the Free Software Foundation, either version 3 of the License, or
@ -16,7 +15,7 @@
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' """
import sqlite3, os import sqlite3, os
from coredb import dbfiles from coredb import dbfiles
import filepaths import filepaths

View File

@ -22,13 +22,19 @@ import os, shutil
import onionrplugins as plugins import onionrplugins as plugins
import logger import logger
import filepaths import filepaths
from utils.readstatic import get_static_dir
def setup_default_plugins(): def setup_default_plugins():
# Copy default plugins into plugins folder # Copy default plugins into plugins folder
if not os.path.exists(plugins.get_plugins_folder()): if not os.path.exists(plugins.get_plugins_folder()):
if os.path.exists('../static-data/default-plugins/'): if os.path.exists(get_static_dir() + '/default-plugins/'):
names = [f for f in os.listdir("../static-data/default-plugins/")] names = [f for f in os.listdir(get_static_dir() + '/default-plugins/')]
shutil.copytree('../static-data/default-plugins/', plugins.get_plugins_folder()) try:
shutil.copytree(
get_static_dir() + '/default-plugins/',
plugins.get_plugins_folder())
except FileExistsError:
pass
# Enable plugins # Enable plugins
for name in names: for name in names:
@ -39,6 +45,8 @@ def setup_default_plugins():
if not os.path.exists(plugins.get_plugin_data_folder(name)): if not os.path.exists(plugins.get_plugin_data_folder(name)):
try: try:
os.mkdir(plugins.get_plugin_data_folder(name)) os.mkdir(plugins.get_plugin_data_folder(name))
except FileExistsError:
pass
except Exception as e: except Exception as e:
#logger.warn('Error enabling plugin: ' + str(e), terminal=True) #logger.warn('Error enabling plugin: ' + str(e), terminal=True)
plugins.disable(name, stop_event = False) plugins.disable(name, stop_event = False)

View File

@ -77,7 +77,7 @@ def store(data, blockHash=''):
raise ValueError('Hash specified does not meet internal hash check') raise ValueError('Hash specified does not meet internal hash check')
else: else:
blockHash = ourHash blockHash = ourHash
if DB_ENTRY_SIZE_LIMIT >= sys.getsizeof(data): if DB_ENTRY_SIZE_LIMIT >= sys.getsizeof(data):
_dbInsert(blockHash, data) _dbInsert(blockHash, data)
else: else:
@ -86,21 +86,23 @@ def store(data, blockHash=''):
def getData(bHash): def getData(bHash):
if not stringvalidators.validate_hash(bHash): raise ValueError if not stringvalidators.validate_hash(bHash): raise ValueError
bHash = bytesconverter.bytes_to_str(bHash) bHash = bytesconverter.bytes_to_str(bHash)
bHash = bHash.strip()
# First check DB for data entry by hash # First check DB for data entry by hash
# if no entry, check disk # if no entry, check disk
# If no entry in either, raise an exception # If no entry in either, raise an exception
retData = None retData = None
fileLocation = '%s/%s.dat' % (filepaths.block_data_location, bHash) fileLocation = '%s/%s.dat' % (filepaths.block_data_location, bHash)
not_found_msg = "Flock data not found for: " not_found_msg = "Block data not found for: "
if os.path.exists(fileLocation): if os.path.exists(fileLocation):
with open(fileLocation, 'rb') as block: with open(fileLocation, 'rb') as block:
retData = block.read() retData = block.read()
else: else:
retData = _dbFetch(bHash) retData = _dbFetch(bHash)
if retData is None: if retData is None:
raise onionrexceptions.NoDataAvailable(not_found_msg + str(bHash)) raise onionrexceptions.NoDataAvailable(not_found_msg + str(bHash))
return retData return retData

View File

@ -1,13 +1,31 @@
import sys, sqlite3 """Onionr - Private P2P Communication.
Test Onionr as it is running
"""
import sys
import sqlite3
import onionrstorage, onionrexceptions, onionrcrypto as crypto import onionrstorage, onionrexceptions, onionrcrypto as crypto
import filepaths import filepaths
from onionrblocks import storagecounter from onionrblocks import storagecounter
from coredb import dbfiles from coredb import dbfiles
from onionrutils import blockmetadata, bytesconverter from onionrutils import blockmetadata, bytesconverter
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def set_data(data)->str: def set_data(data)->str:
''' """Set the data assciated with a hash."""
Set the data assciated with a hash
'''
storage_counter = storagecounter.StorageCounter() storage_counter = storagecounter.StorageCounter()
data = data data = data
dataSize = sys.getsizeof(data) dataSize = sys.getsizeof(data)

View File

@ -11,6 +11,7 @@ from . import uicheck, inserttest, stresstest
from . import ownnode from . import ownnode
from .webpasstest import webpass_test from .webpasstest import webpass_test
from .osver import test_os_ver_endpoint from .osver import test_os_ver_endpoint
from .clearnettor import test_clearnet_tor_request
""" """
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
@ -32,7 +33,8 @@ RUN_TESTS = [uicheck.check_ui,
ownnode.test_own_node, ownnode.test_own_node,
stresstest.stress_test_block_insert, stresstest.stress_test_block_insert,
webpass_test, webpass_test,
test_os_ver_endpoint test_os_ver_endpoint,
test_clearnet_tor_request
] ]
SUCCESS_FILE = os.path.dirname(os.path.realpath(__file__)) + '/../../tests/runtime-result.txt' SUCCESS_FILE = os.path.dirname(os.path.realpath(__file__)) + '/../../tests/runtime-result.txt'

View File

@ -0,0 +1,61 @@
"""Onionr - Private P2P Communication.
Ensure that clearnet cannot be reached
"""
from threading import Thread
from onionrutils.basicrequests import do_get_request
from onionrutils import localcommand
import logger
import config
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def test_clearnet_tor_request(testmanager):
"""Ensure that Tor cannot request clearnet address.
Does not run if Tor is being reused
"""
config.reload()
leak_result = ""
if config.get('tor.use_existing_tor', False):
logger.warn(
"Can't ensure Tor reqs to clearnet won't happen when reusing Tor")
return
socks_port = localcommand.local_command('/gettorsocks')
# Don't worry, this request isn't meant to go through,
# but if it did it would be through Tor
try:
leak_result: str = do_get_request(
'https://onionr.net/404',
port=socks_port, ignoreAPI=True).lower()
except AttributeError:
leak_result = ""
except Exception as e:
logger.warn(str(e))
try:
if 'not found' in leak_result:
logger.error('Tor was able to request a clearnet site')
raise ValueError('Tor was able to request a clearnet site')
except TypeError:
pass

View File

@ -11,7 +11,7 @@ def _check_remote_node(testmanager):
def insert_bin_test(testmanager): def insert_bin_test(testmanager):
data = os.urandom(32) data = os.urandom(32)
b_hash = onionrblocks.insert(data, ) b_hash = onionrblocks.insert(data)
if not b_hash in coredb.blockmetadb.get_block_list(): if not b_hash in coredb.blockmetadb.get_block_list():
logger.error(str(b_hash) + 'is not in bl') logger.error(str(b_hash) + 'is not in bl')

View File

@ -25,14 +25,16 @@ from onionrutils import localcommand
def test_own_node(test_manager): def test_own_node(test_manager):
return
socks_port = localcommand.local_command('/gettorsocks') socks_port = localcommand.local_command('/gettorsocks')
if config.get('general.security_level', 0) > 0: if config.get('general.security_level', 0) > 0:
return return
own_tor_address = gettransports.get()[0] own_tor_address = gettransports.get()[0]
if 'this is an onionr node' \ if 'this is an onionr node' \
not in basicrequests.do_get_request('http://' + own_tor_address, not in basicrequests.do_get_request('http://' + own_tor_address,
port=socks_port, ignoreAPI=True).lower(): port=socks_port,
logger.warn('Own node not reachable in test') ignoreAPI=True).lower():
logger.warn(f'Own node not reachable in test {own_tor_address}')
raise ValueError raise ValueError

View File

@ -1,4 +1,4 @@
{ "name": "flow", { "name": "flow",
"version": "0.0.1", "version": "0.1.0",
"author": "onionr" "author": "onionr"
} }

View File

@ -39,8 +39,9 @@ flask_blueprint = flowapi.flask_blueprint
security_whitelist = ['staticfiles.boardContent', 'staticfiles.board'] security_whitelist = ['staticfiles.boardContent', 'staticfiles.board']
plugin_name = 'flow' plugin_name = 'flow'
PLUGIN_VERSION = '0.0.1' PLUGIN_VERSION = '0.1.0'
EXPIRE_TIME = 43200
class OnionrFlow: class OnionrFlow:
def __init__(self): def __init__(self):
@ -75,7 +76,7 @@ class OnionrFlow:
else: else:
if message == "q": if message == "q":
self.flowRunning = False self.flowRunning = False
expireTime = epoch.get_epoch() + 43200 expireTime = epoch.get_epoch() + EXPIRE_TIME
if len(message) > 0: if len(message) > 0:
logger.info('Inserting message as block...', terminal=True) logger.info('Inserting message as block...', terminal=True)
onionrblocks.insert(message, header='brd', onionrblocks.insert(message, header='brd',
@ -118,6 +119,24 @@ def on_flow_cmd(api, data=None):
OnionrFlow().start() OnionrFlow().start()
def on_flowsend_cmd(api, data=None):
err_msg = "Second arg is board name, third is quoted message"
try:
sys.argv[2]
except IndexError:
logger.error(err_msg, terminal=True)
try:
sys.argv[3]
except IndexError:
logger.error(err_msg, terminal=True)
bl = onionrblocks.insert(sys.argv[3], header='brd',
expire=(EXPIRE_TIME + epoch.get_epoch()),
meta={'ch': sys.argv[2]})
print(bl)
def on_softreset(api, data=None): def on_softreset(api, data=None):
try: try:
os.remove(identifyhome.identify_home() + '/board-index.cache.json') os.remove(identifyhome.identify_home() + '/board-index.cache.json')

View File

@ -46,7 +46,9 @@
"minimum_score": -100 "minimum_score": -100
}, },
"plugins": { "plugins": {
"disabled": [], "disabled": [
"chat"
],
"enabled": [] "enabled": []
}, },
"timers": { "timers": {

View File

@ -0,0 +1,18 @@
import sys
import os
from subprocess import Popen, PIPE
import uuid
TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
os.environ["ONIONR_HOME"] = TEST_DIR
print(f'running integration test for {__file__}')
with Popen(['./onionr.sh', 'details'], stdout=PIPE) as onionr_proc:
output = onionr_proc.stdout.read().decode()
if onionr_proc.returncode != 0:
raise ValueError('Raised non zero exit ' + str(onionr_proc.returncode))
for word in ['Node', 'Human-readable']:
if word not in output:
raise ValueError(word + " not in " + output)

View File

@ -0,0 +1,40 @@
from unittest.mock import patch
import sys, os
sys.path.append(".")
sys.path.append("src/")
import unittest, uuid
TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
print("Test directory:", TEST_DIR)
os.environ["ONIONR_HOME"] = TEST_DIR
from utils import createdirs
from onionrcommands import parser
import onionrsetup as setup
from netcontroller.torcontrol import customtorrc
from utils import createdirs
from onionrsetup import setup_config, setup_default_plugins
from coredb import blockmetadb
from etc.onionrvalues import BLOCK_EXPORT_FILE_EXT
createdirs.create_dirs()
setup_config()
setup_default_plugins()
import config
from filepaths import export_location
class OnionrTests(unittest.TestCase):
def test_export(self):
testargs = ["onionr.py", "flowsend", "tests", "hello"]
with patch.object(sys, 'argv', testargs):
parser.register()
bl = blockmetadb.get_block_list()[0]
testargs = ["onionr.py", "export-block", bl]
with patch.object(sys, 'argv', testargs):
parser.register()
with open(export_location + '/' + bl + BLOCK_EXPORT_FILE_EXT, 'rb') as f:
if b'hello' not in f.read():
raise ValueError('No exported block')
unittest.main()

View File

@ -0,0 +1,16 @@
import sys
import os
from subprocess import Popen, PIPE
import uuid
TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
os.environ["ONIONR_HOME"] = TEST_DIR
print(f'running integration test for {__file__}')
with Popen(['./onionr.sh'], stdout=PIPE) as onionr_proc:
output = onionr_proc.stdout.read().decode()
if onionr_proc.returncode != 0:
raise ValueError('Raised non zero exit ' + str(onionr_proc.returncode))
if output != '':
raise ValueError('No command run returned non-blank output')

View File

@ -1 +1 @@
1580971981 1581152327