fixed housekeeping being broken

scripts/announce-attack.py

@@ -11,7 +11,7 @@ from onionrutils import basicrequests
 def random_tor_generator():
     return base64.b32encode(secrets.token_bytes(35)).decode().replace("=", "").lower() + ".onion"
 
-node = input("Enter node to attack. Note that you must legally use your own, and even that might lead to issues")
+node = input("Enter node to attack. Note that you legally must use your own, and even that might lead to technical or legal issues")
 assert stringvalidators.validate_transport(node)
 
 count = int(input("Attack amount: "))

scripts/block-spammer.py

@@ -4,7 +4,7 @@
 
 # Please don't run this script on the real Onionr network. You wouldn't do anything but be annoying
 
-print("Please don't run this script on the real Onionr network. You wouldn't do anything but be annoying, and possibly violate law")
+print("Please don't run this script on Onionr networks that include more than you. You wouldn't do anything but be annoying, and probably violate law")
 
 import sys
 import os
@@ -14,8 +14,17 @@ import onionrblocks
 
 
 amount = int(input("Number of blocks:"))
+expire = input("Expire in seconds:")
+
+if not expire:
+    expire = ""
+else:
+    expire = int(expire)
 
 for i in range(amount):
-    onionrblocks.insert(data=os.urandom(32))
+    if expire:
+        print(onionrblocks.insert(data=os.urandom(32), expire=expire))
+    else:
+        print(onionrblocks.insert(data=os.urandom(32)))
     print(i, "done")
 

scripts/disable-dev-config.py

@@ -18,6 +18,7 @@ conf['general']['display_header'] = True
 conf['onboarding']['done'] = False
 conf['general']['minimum_block_pow'] = 5
 conf['general']['minimum_send_pow'] = 5
+conf['general']['max_block_age'] = 2678400
 conf['log']['file']['remove_on_exit'] = True
 
 json.dump(conf, open('static-data/default_config.json', 'w'), sort_keys=True, indent=4)

scripts/enable-dev-config.py

@@ -6,6 +6,8 @@ import json
 
 conf = json.load(open('static-data/default_config.json', 'r'))
 
+block_pow = int(input("Block POW level:"))
+
 if input("Reuse Tor? y/n:").lower() == 'y':
     conf['tor']['use_existing_tor'] = True
     conf['tor']['existing_control_port'] = int(input("Enter existing control port:"))
@@ -16,8 +18,8 @@ conf['general']['dev_mode'] = True
 conf['general']['insert_deniable_blocks'] = False
 conf['general']['random_bind_ip'] = False
 conf['onboarding']['done'] = True
-conf['general']['minimum_block_pow'] = 4
-conf['general']['minimum_send_pow'] = 4
+conf['general']['minimum_block_pow'] = block_pow
+conf['general']['minimum_send_pow'] = block_pow
 conf['log']['file']['remove_on_exit'] = False
 
 json.dump(conf, open('static-data/default_config.json', 'w'), sort_keys=True, indent=4)

src/communicatorutils/housekeeping.py

@@ -44,11 +44,14 @@ def clean_old_blocks(comm_inst):
         logger.info('Deleted block: %s' % (bHash,))
 
     while comm_inst.storage_counter.is_full():
-        oldest = blockmetadb.get_block_list()[0]
+        try:
+            oldest = blockmetadb.get_block_list()[0]
+        except IndexError:
+            break
         blacklist.addToDB(oldest)
         removeblock.remove_block(oldest)
         onionrstorage.deleteBlock(oldest)
-        __remove_from_upload.remove(comm_inst, oldest)
+        __remove_from_upload(comm_inst, oldest)
         logger.info('Deleted block: %s' % (oldest,))
 
     comm_inst.decrementThreadCount('clean_old_blocks')

src/coredb/blockmetadb/add.py

@@ -1,9 +1,15 @@
-'''
-    Onionr - Private P2P Communication
+"""Onionr - Private P2P Communication.
 
-    Add an entry to the block metadata database
-'''
-'''
+Add an entry to the block metadata database
+"""
+import os
+import sqlite3
+import secrets
+from onionrutils import epoch, blockmetadata
+from etc import onionrvalues
+from .. import dbfiles
+from onionrexceptions import BlockMetaEntryExists
+"""
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation, either version 3 of the License, or
@@ -16,20 +22,18 @@
 
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import os, sqlite3, secrets
-from onionrutils import epoch, blockmetadata
-from etc import onionrvalues
-from .. import dbfiles
+"""
+
+
 def add_to_block_DB(newHash, selfInsert=False, dataSaved=False):
-    '''
+    """
         Add a hash value to the block db
 
         Should be in hex format!
-    '''
+    """
 
     if blockmetadata.has_block(newHash):
-        return
+        raise
     conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
     c = conn.cursor()
     currentTime = epoch.get_epoch() + secrets.randbelow(301)
@@ -40,4 +44,4 @@ def add_to_block_DB(newHash, selfInsert=False, dataSaved=False):
     data = (newHash, currentTime, '', selfInsert)
     c.execute('INSERT INTO hashes (hash, dateReceived, dataType, dataSaved) VALUES(?, ?, ?, ?);', data)
     conn.commit()
-    conn.close()
+    conn.close()

src/etc/onionrvalues.py

@@ -42,7 +42,6 @@ DATABASE_LOCK_TIMEOUT = 60
 
 # Block creation anonymization requirements
 MIN_BLOCK_UPLOAD_PEER_PERCENT = 0.1
-MIN_SHARE_WAIT_DELAY_SECS = 5
 
 WSGI_SERVER_REQUEST_TIMEOUT_SECS = 120
 
@@ -55,7 +54,8 @@ BLOCK_EXPORT_FILE_EXT = '.dat'
 """30 days is plenty of time for someone to decide to renew a block"""
 DEFAULT_EXPIRE = 2678400
 # Metadata header section length limits, in bytes
-BLOCK_METADATA_LENGTHS = {'meta': 1000, 'sig': 200, 'signer': 200, 'time': 10, 'pow': 1000, 'encryptType': 4, 'expire': 14}
+BLOCK_METADATA_LENGTHS = {'meta': 1000, 'sig': 200, 'signer': 200, 'time': 10,
+                          'pow': 1000, 'encryptType': 4, 'expire': 14}
 
 # Pool Eligibility Max Age
 BLOCK_POOL_MAX_AGE = 300

src/onionrblocks/insert/main.py

@@ -62,6 +62,8 @@ def insert_block(data: Union[str, bytes], header: str = 'txt',
     """
         Inserts a block into the network
         encryptType must be specified to encrypt a block
+        if expire is less than date, assumes seconds into future.
+            if not assume exact epoch
     """
     our_private_key = crypto.priv_key
     our_pub_key = crypto.pub_key
@@ -180,6 +182,9 @@ def insert_block(data: Union[str, bytes], header: str = 'txt',
     # ensure expire is integer and of sane length
     if type(expire) is not type(None):
         if not len(str(int(expire))) < 20: raise ValueError('expire must be valid int less than 20 digits in length')
+        # if expire is less than date, assume seconds into future
+        if expire < epoch.get_epoch():
+            expire = epoch.get_epoch() + expire
         metadata['expire'] = expire
 
     # send block data (and metadata) to POW module to get tokenized block data
@@ -207,8 +212,14 @@ def insert_block(data: Union[str, bytes], header: str = 'txt',
             coredb.blockmetadb.add.add_to_block_DB(retData, selfInsert=True, dataSaved=True)
 
             if expire is None:
-                coredb.blockmetadb.update_block_info(retData, 'expire',
-                                                     createTime + onionrvalues.DEFAULT_EXPIRE)
+                coredb.blockmetadb.update_block_info(
+                    retData, 'expire',
+                    createTime +
+                    min(
+                        onionrvalues.DEFAULT_EXPIRE,
+                        config.get(
+                            'general.max_block_age',
+                            onionrvalues.DEFAULT_EXPIRE)))
             else:
                 coredb.blockmetadb.update_block_info(retData, 'expire', expire)
 

src/onionrexceptions.py

@@ -54,7 +54,7 @@ class ReplayAttack(Exception):
 class InvalidUpdate(Exception):
     pass
 
-class DifficultyTooLarge(Exception):
+class BlockMetaEntryExists(Exception):
     pass
 
 class InvalidMetadata(Exception):

static-data/default_config.json

@@ -3,8 +3,6 @@
         "security_auditing": true
     },
     "allocations": {
-        "blockCache": 5000000,
-        "blockCacheTotal": 50000000,
         "disk": 100000000,
         "net_total": 1000000000
     },

tests/runtime-result.txt

@@ -1 +1 @@
-1582946012
+1583020786

tests/test_onionrvalues.py

@@ -15,5 +15,10 @@ class TestOnionrValues(unittest.TestCase):
     def test_default_expire(self):
         self.assertEqual(onionrvalues.DEFAULT_EXPIRE, 2678400)
 
+    def test_block_clock_skew(self):
+        self.assertEqual(onionrvalues.MAX_BLOCK_CLOCK_SKEW, 120)
+
+    def test_block_export_ext(self):
+        self.assertEqual(onionrvalues.BLOCK_EXPORT_FILE_EXT, '.dat')
 
 unittest.main()