added sneakernet auto importing

exportblocks now takes argument

.vscode/tasks.json

@@ -46,6 +46,13 @@
         "command": "scripts/disable-dev-config.py",
         "group": "dev",
         "problemMatcher": []
+      },
+      {
+        "label": "Format default config",
+        "type": "process",
+        "command": "scripts/pretty-default-config.py",
+        "group": "dev",
+        "problemMatcher": []
       }
     ]
   }

requirements.in

@@ -8,9 +8,9 @@ stem==1.8.0
 deadsimplekv==0.3.1
 unpaddedbase32==0.2.0
 streamedrequests==1.0.0
-jinja2==2.11.1
 toomanyobjs==1.1.0
 niceware==0.2.1
 psutil==5.7.0
 filenuke==0.0.0
 mimcvdf==1.0.0
+watchdog==0.10.2

requirements.txt

@@ -115,7 +115,7 @@ itsdangerous==1.1.0 \
 jinja2==2.11.1 \
     --hash=sha256:93187ffbc7808079673ef52771baa950426fd664d3aad1d0fa3e95644360e250 \
     --hash=sha256:b0eaf100007721b5c16c1fc1eecb87409464edc10469ddc9a22a27a99123be49 \
-    # via -r requirements.in, flask
+    # via flask
 markupsafe==1.1.1 \
     --hash=sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473 \
     --hash=sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161 \
@@ -154,6 +154,9 @@ niceware==0.2.1 \
     --hash=sha256:0f8b192f2a1e800e068474f6e208be9c7e2857664b33a96f4045340de4e5c69c \
     --hash=sha256:cf2dc0e1567d36d067c61b32fed0f1b9c4534ed511f9eeead4ba548d03b5c9eb \
     # via -r requirements.in
+pathtools==0.1.2 \
+    --hash=sha256:7c35c5421a39bb82e58018febd90e3b6e5db34c5443aaaf742b3f33d4655f1c0 \
+    # via watchdog
 psutil==5.7.0 \
     --hash=sha256:1413f4158eb50e110777c4f15d7c759521703bd6beb58926f1d562da40180058 \
     --hash=sha256:298af2f14b635c3c7118fd9183843f4e73e681bb6f01e12284d4d70d48a60953 \
@@ -221,6 +224,9 @@ urllib3==1.25.8 \
     --hash=sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc \
     --hash=sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc \
     # via -r requirements.in, requests
+watchdog==0.10.2 \
+    --hash=sha256:c560efb643faed5ef28784b2245cf8874f939569717a4a12826a173ac644456b \
+    # via -r requirements.in
 werkzeug==0.15.5 \
     --hash=sha256:87ae4e5b5366da2347eb3116c0e6c681a0e939a33b2805e2c0cbd282664932c4 \
     --hash=sha256:a13b74dd3c45f758d4ebdb224be8f1ab8ef58b3c0ffc1783a8c7d9f4f50227e6 \

scripts/disable-dev-config.py

@@ -23,7 +23,7 @@ conf['general']['max_block_age'] = 2678400
 conf['log']['file']['remove_on_exit'] = True
 conf['transports']['lan'] = True
 conf['transports']['tor'] = True
-conf['transports']['manual_disk'] = True
+conf['transports']['sneakernet'] = True
 
 json.dump(conf, open('static-data/default_config.json', 'w'), sort_keys=True, indent=4)
 

src/etc/onionrvalues.py

@@ -47,7 +47,7 @@ WSGI_SERVER_REQUEST_TIMEOUT_SECS = 120
 
 MAX_NEW_PEER_QUEUE = 1000
 
-BLOCK_EXPORT_FILE_EXT = '.dat'
+BLOCK_EXPORT_FILE_EXT = '.onionr'
 
 # Begin OnionrValues migrated values
 

src/onionrcommands/daemonlaunch/__init__.py

@@ -37,6 +37,7 @@ from .killdaemon import kill_daemon  # noqa
 from utils.boxprint import bordered
 from lan import LANManager
 from lan.server import LANServer
+from sneakernet import sneakernet_import_thread
 """
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -169,6 +170,8 @@ def daemon():
         Thread(target=LANServer(shared_state).start_server,
                daemon=True).start()
         LANManager(shared_state).start()
+    if config.get('transports.sneakernet', True):
+        Thread(target=sneakernet_import_thread, daemon=True).start()
     communicator.startCommunicator(shared_state)
 
     clean_ephemeral_services()

src/onionrcommands/exportblocks.py

@@ -8,6 +8,7 @@ import logger
 import onionrstorage
 from utils import createdirs
 from onionrutils import stringvalidators
+from etc.onionrvalues import BLOCK_EXPORT_FILE_EXT
 import filepaths
 
 import os
@@ -31,23 +32,26 @@ from coredb import blockmetadb
 def _do_export(b_hash):
     createdirs.create_dirs()
     data = onionrstorage.getData(b_hash)
-    with open('%s/%s.dat' % (filepaths.export_location,
-                             b_hash), 'wb') as export:
+    with open('%s/%s%s' % (filepaths.export_location,
+                             b_hash, BLOCK_EXPORT_FILE_EXT), 'wb') as export:
         export.write(data)
         logger.info('Block exported as file', terminal=True)
 
 
-def export_block():
+def export_block(*args):
     """Export block based on hash from stdin or argv."""
-    try:
-        if not stringvalidators.validate_hash(sys.argv[2]):
-            raise ValueError
-    except (IndexError, ValueError):
-        logger.error('No valid block hash specified.', terminal=True)
-        sys.exit(1)
+    if args:
+        b_hash = args[0]
     else:
-        b_hash = sys.argv[2]
-        _do_export(b_hash)
+        try:
+            if not stringvalidators.validate_hash(sys.argv[2]):
+                raise ValueError
+        except (IndexError, ValueError):
+            logger.error('No valid block hash specified.', terminal=True)
+            sys.exit(1)
+        else:
+            b_hash = sys.argv[2]
+    _do_export(b_hash)
 
 
 export_block.onionr_help = "<block hash>: Export block to "  # type: ignore

src/onionrutils/importnewblocks.py

@@ -23,6 +23,7 @@ from onionrutils import blockmetadata
 from coredb import blockmetadb
 import filepaths
 import onionrcrypto as crypto
+from etc.onionrvalues import BLOCK_EXPORT_FILE_EXT
 def import_new_blocks(scanDir=''):
     '''
         This function is intended to scan for new blocks ON THE DISK and import them
@@ -33,19 +34,19 @@ def import_new_blocks(scanDir=''):
         scanDir = filepaths.block_data_location
     if not scanDir.endswith('/'):
         scanDir += '/'
-    for block in glob.glob(scanDir + "*.dat"):
-        if block.replace(scanDir, '').replace('.dat', '') not in blockList:
+    for block in glob.glob(scanDir + "*%s" % (BLOCK_EXPORT_FILE_EXT,)):
+        if block.replace(scanDir, '').replace(BLOCK_EXPORT_FILE_EXT, '') not in blockList:
             exist = True
             logger.info('Found new block on dist %s' % block, terminal=True)
             with open(block, 'rb') as newBlock:
-                block = block.replace(scanDir, '').replace('.dat', '')
-                if crypto.hashers.sha3_hash(newBlock.read()) == block.replace('.dat', ''):
-                    blockmetadb.add_to_block_DB(block.replace('.dat', ''), dataSaved=True)
-                    logger.info('Imported block %s.' % block, terminal=True)
+                block = block.replace(scanDir, '').replace(BLOCK_EXPORT_FILE_EXT, '')
+                if crypto.hashers.sha3_hash(newBlock.read()) == block.replace(BLOCK_EXPORT_FILE_EXT, ''):
+                    blockmetadb.add_to_block_DB(block.replace(BLOCK_EXPORT_FILE_EXT, ''), dataSaved=True)
+                    logger.info('Imported block %s' % block, terminal=True)
                     blockmetadata.process_block_metadata(block)
                 else:
                     logger.warn('Failed to verify hash for %s' % block, terminal=True)
     if not exist:
         logger.info('No blocks found to import', terminal=True)
 
-import_new_blocks.onionr_help = f"Scans the Onionr data directory under {filepaths.block_data_location} for new block files (.dat, .db not supported) to import"
+import_new_blocks.onionr_help = f"Scans the Onionr data directory under {filepaths.block_data_location} for new block files (.db not supported) to import"

src/runtests/__init__.py

@@ -15,6 +15,7 @@ from .osver import test_os_ver_endpoint
 from .clearnettor import test_clearnet_tor_request
 from .housekeeping import test_inserted_housekeeping
 from .lanservertest import test_lan_server
+from .sneakernettest import test_sneakernet_import
 """
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -39,7 +40,8 @@ RUN_TESTS = [uicheck.check_ui,
              test_os_ver_endpoint,
              test_clearnet_tor_request,
              test_inserted_housekeeping,
-             test_lan_server
+             test_lan_server,
+             sneakernettest.test_sneakernet_import
              ]
 
 SUCCESS_FILE = os.path.dirname(os.path.realpath(__file__)) + '/../../tests/runtime-result.txt'

src/runtests/sneakernettest.py

@@ -0,0 +1,27 @@
+import os
+from shutil import move
+
+from onionrblocks import insert
+from onionrstorage import deleteBlock
+from onionrcommands.exportblocks import export_block
+from filepaths import export_location, block_data_location, data_nonce_file
+from etc.onionrvalues import BLOCK_EXPORT_FILE_EXT
+from onionrstorage.removeblock import remove_block
+from onionrstorage import deleteBlock
+from coredb.blockmetadb import get_block_list
+from utils import bettersleep
+from gevent import sleep
+
+def test_sneakernet_import(test_manager):
+    in_db = lambda b: b in get_block_list()
+    bl = insert(os.urandom(10))
+    assert in_db(bl)
+    export_block(bl)
+    assert os.path.exists(export_location + bl + BLOCK_EXPORT_FILE_EXT)
+    remove_block(bl)
+    deleteBlock(bl)
+    assert not in_db(bl)
+    os.remove(data_nonce_file)
+    move(export_location + bl + BLOCK_EXPORT_FILE_EXT, block_data_location)
+    sleep(1)
+    assert in_db(bl)

src/sneakernet/__init__.py

@@ -0,0 +1,54 @@
+"""Onionr - Private P2P Communication.
+
+Detect new block files in a given directory
+"""
+import os
+
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler
+
+import config
+from filepaths import block_data_location
+from etc.onionrvalues import BLOCK_EXPORT_FILE_EXT
+from onionrblocks.blockimporter import import_block_from_data
+"""
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
+watch_paths = config.get('transports.sneakernet.paths', list([]))
+if block_data_location not in watch_paths:
+    watch_paths.append(block_data_location)
+
+
+class _Importer(FileSystemEventHandler):
+    @staticmethod
+    def on_created(event):
+        if not event.src_path.endswith(BLOCK_EXPORT_FILE_EXT):
+            return
+        with open(event.src_path, 'rb') as block_file:
+            import_block_from_data(block_file.read())
+            if block_data_location in event.src_path:
+                try:
+                    os.remove(event.src_path)
+                except FileNotFoundError:
+                    pass
+
+
+def sneakernet_import_thread():
+    observer = Observer()
+    for path in watch_paths:
+        observer.schedule(_Importer(), path, recursive=True)
+    observer.start()
+    while observer.isAlive():
+        observer.join(60)

static-data/default-plugins/metadataprocessor/info.json

@@ -1,5 +0,0 @@
-{
-    "name" : "metadataprocessor",
-    "version" : "1.0",
-    "author" : "onionr"
-}

static-data/default-plugins/metadataprocessor/main.py

@@ -1,59 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    This processes metadata for Onionr blocks
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-
-# useful libraries
-import logger, config
-import os, sys, json, time, random, shutil, base64, getpass, datetime, re
-import onionrusers, onionrexceptions
-from onionrutils import stringvalidators
-
-plugin_name = 'metadataprocessor'
-
-# event listeners
-
-def _processForwardKey(api, myBlock):
-    '''
-        Get the forward secrecy key specified by the user for us to use
-    '''
-    peer = onionrusers.OnionrUser(myBlock.signer)
-    key = myBlock.getMetadata('newFSKey')
-
-    # We don't need to validate here probably, but it helps
-    if stringvalidators.validate_pub_key(key):
-        peer.addForwardKey(key)
-    else:
-        raise onionrexceptions.InvalidPubkey("%s is not a valid pubkey key" % (key,))
-
-def on_processblocks(api, data=None):
-    # Generally fired by utils.
-    myBlock = api.data['block']
-    blockType = api.data['type']
-    # Process specific block types
-
-    # forwardKey blocks, add a new forward secrecy key for a peer
-    if blockType == 'forwardKey':
-        if api.data['validSig'] == True:
-            _processForwardKey(api, myBlock)
-    
-def on_init(api, data = None):
-
-    pluginapi = api
-
-    return

static-data/default_config.json

@@ -63,7 +63,7 @@
     },
     "transports": {
         "lan": true,
-        "manual_disk": true,
+        "sneakernet": true,
         "tor": true
     },
     "ui": {

tests/runtime-result.txt

@@ -1 +1 @@
-1585525008
+1585619396

tests/test_default_config_json.py

@@ -58,7 +58,7 @@ class OnionrConfig(unittest.TestCase):
             self.assertEqual(conf['tor']['use_bridge'], False)
             self.assertEqual(conf['tor']['use_existing_tor'], False)
             self.assertEqual(conf['transports']['lan'], True)
-            self.assertEqual(conf['transports']['manual_disk'], True)
+            self.assertEqual(conf['transports']['sneakernet'], True)
             self.assertEqual(conf['transports']['tor'], True)
             self.assertEqual(conf['ui']['theme'], 'dark')
 unittest.main()

tests/test_onionrvalues.py

@@ -19,6 +19,6 @@ class TestOnionrValues(unittest.TestCase):
         self.assertEqual(onionrvalues.MAX_BLOCK_CLOCK_SKEW, 120)
 
     def test_block_export_ext(self):
-        self.assertEqual(onionrvalues.BLOCK_EXPORT_FILE_EXT, '.dat')
+        self.assertEqual(onionrvalues.BLOCK_EXPORT_FILE_EXT, '.onionr')
 
 unittest.main()