From 8082570b7fc78470d601b0d57171a7b8b723805e Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Wed, 19 Jun 2019 01:57:13 -0500
Subject: [PATCH 01/29] * bumped nacl and unpaddedbase32 verison *
 added/improved support for unpaddedbase32 keys * greatly improved home UI and
 mail * deniable blocks shouldnt use forward secrecy anymore * dont add
 yourself as a contact

---
 onionr/communicatorutils/deniableinserts.py  |  2 +-
 onionr/core.py                               |  9 +++-
 onionr/httpapi/friendsapi/__init__.py        |  3 +-
 onionr/onionrcommands/pubkeymanager.py       |  2 +
 onionr/onionrcrypto.py                       |  6 ++-
 onionr/onionrusers/contactmanager.py         |  2 +
 onionr/onionrusers/onionrusers.py            |  5 +-
 onionr/onionrutils.py                        |  6 ++-
 onionr/static-data/www/mail/mail.css         |  4 ++
 onionr/static-data/www/mail/mail.js          |  6 +--
 onionr/static-data/www/mail/sendmail.js      |  7 ++-
 onionr/static-data/www/private/index.html    | 10 ++--
 onionr/static-data/www/private/main.css      | 10 ++++
 onionr/static-data/www/shared/main/stats.js  | 25 +++++++++
 onionr/static-data/www/shared/main/style.css |  8 +++
 onionr/tests/test_stringvalidations.py       |  8 +--
 requirements.in                              |  3 +-
 requirements.txt                             | 56 ++++++++------------
 18 files changed, 117 insertions(+), 55 deletions(-)

diff --git a/onionr/communicatorutils/deniableinserts.py b/onionr/communicatorutils/deniableinserts.py
index f993e6d0..571a6093 100755
--- a/onionr/communicatorutils/deniableinserts.py
+++ b/onionr/communicatorutils/deniableinserts.py
@@ -27,5 +27,5 @@ def insert_deniable_block(comm_inst):
         # This assumes on the libsodium primitives to have key-privacy
         fakePeer = onionrvalues.DENIABLE_PEER_ADDRESS
         data = secrets.token_hex(secrets.randbelow(1024) + 1)
-        comm_inst._core.insertBlock(data, header='pm', encryptType='asym', asymPeer=fakePeer, meta={'subject': 'foo'})
+        comm_inst._core.insertBlock(data, header='pm', encryptType='asym', asymPeer=fakePeer, disableForward=True, meta={'subject': 'foo'})
     comm_inst.decrementThreadCount('insert_deniable_block')
\ No newline at end of file
diff --git a/onionr/core.py b/onionr/core.py
index 33b310af..be309309 100755
--- a/onionr/core.py
+++ b/onionr/core.py
@@ -128,7 +128,8 @@ class Core:
         '''
             Adds a public key to the key database (misleading function name)
         '''
-        assert peerID not in self.listPeers()
+        if peerID in self.listPeers() or peerID == self._crypto.pubKey:
+            raise ValueError("specified id is already known")
 
         # This function simply adds a peer to the DB
         if not self._utils.validatePubKey(peerID):
@@ -776,7 +777,11 @@ class Core:
                 data = self._crypto.pubKeyEncrypt(data, asymPeer, encodedData=True).decode()
                 signature = self._crypto.pubKeyEncrypt(signature, asymPeer, encodedData=True).decode()
                 signer = self._crypto.pubKeyEncrypt(signer, asymPeer, encodedData=True).decode()
-                onionrusers.OnionrUser(self, asymPeer, saveUser=True)
+                try:
+                    onionrusers.OnionrUser(self, asymPeer, saveUser=True)
+                except ValueError:
+                    # if peer is already known
+                    pass
             else:
                 raise onionrexceptions.InvalidPubkey(asymPeer + ' is not a valid base32 encoded ed25519 key')
 
diff --git a/onionr/httpapi/friendsapi/__init__.py b/onionr/httpapi/friendsapi/__init__.py
index 8d6b4122..5b4a883b 100755
--- a/onionr/httpapi/friendsapi/__init__.py
+++ b/onionr/httpapi/friendsapi/__init__.py
@@ -26,7 +26,8 @@ friends = Blueprint('friends', __name__)
 @friends.route('/friends/list')
 def list_friends():
     pubkey_list = {}
-    friend_list = contactmanager.ContactManager.list_friends(core.Core())
+    c = core.Core()
+    friend_list = contactmanager.ContactManager.list_friends(c)
     for friend in friend_list:
         pubkey_list[friend.publicKey] = {'name': friend.get_info('name')}
     return json.dumps(pubkey_list)
diff --git a/onionr/onionrcommands/pubkeymanager.py b/onionr/onionrcommands/pubkeymanager.py
index e12fc670..2a6370f9 100755
--- a/onionr/onionrcommands/pubkeymanager.py
+++ b/onionr/onionrcommands/pubkeymanager.py
@@ -21,6 +21,7 @@
 import sys, getpass
 import logger, onionrexceptions
 from onionrusers import onionrusers, contactmanager
+import unpaddedbase32
 def add_ID(o_inst):
     try:
         sys.argv[2]
@@ -50,6 +51,7 @@ def add_ID(o_inst):
 def change_ID(o_inst):
     try:
         key = sys.argv[2]
+        key = unpaddedbase32.repad(key.encode()).decode()
     except IndexError:
         logger.warn('Specify pubkey to use')
     else:
diff --git a/onionr/onionrcrypto.py b/onionr/onionrcrypto.py
index 75fe2138..3c77b3ce 100755
--- a/onionr/onionrcrypto.py
+++ b/onionr/onionrcrypto.py
@@ -19,6 +19,7 @@
 '''
 import os, binascii, base64, hashlib, time, sys, hmac, secrets
 import nacl.signing, nacl.encoding, nacl.public, nacl.hash, nacl.pwhash, nacl.utils, nacl.secret
+import unpaddedbase32
 import logger, onionrproofs
 import onionrexceptions, keymanager, core
 import config
@@ -93,6 +94,7 @@ class OnionrCrypto:
 
     def pubKeyEncrypt(self, data, pubkey, encodedData=False):
         '''Encrypt to a public key (Curve25519, taken from base32 Ed25519 pubkey)'''
+        pubkey = unpaddedbase32.repad(self._core._utils.strToBytes(pubkey))
         retVal = ''
         box = None
         data = self._core._utils.strToBytes(data)
@@ -129,7 +131,7 @@ class OnionrCrypto:
         return decrypted
 
     def symmetricEncrypt(self, data, key, encodedKey=False, returnEncoded=True):
-        '''Encrypt data to a 32-byte key (Salsa20-Poly1305 MAC)'''
+        '''Encrypt data with a 32-byte key (Salsa20-Poly1305 MAC)'''
         if encodedKey:
             encoding = nacl.encoding.Base64Encoder
         else:
@@ -199,7 +201,7 @@ class OnionrCrypto:
         if pubkey == '':
             pubkey = self.pubKey
         prev = ''
-        pubkey = pubkey.encode()
+        pubkey = self._core._utils.strToBytes(pubkey)
         for i in range(self.HASH_ID_ROUNDS):
             try:
                 prev = prev.encode()
diff --git a/onionr/onionrusers/contactmanager.py b/onionr/onionrusers/contactmanager.py
index 0524ed64..098cba65 100755
--- a/onionr/onionrusers/contactmanager.py
+++ b/onionr/onionrusers/contactmanager.py
@@ -18,10 +18,12 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 import os, json, onionrexceptions
+import unpaddedbase32
 from onionrusers import onionrusers
 
 class ContactManager(onionrusers.OnionrUser):
     def __init__(self, coreInst, publicKey, saveUser=False, recordExpireSeconds=5):
+        publicKey = unpaddedbase32.repad(coreInst._utils.strToBytes(publicKey)).decode()
         super(ContactManager, self).__init__(coreInst, publicKey, saveUser=saveUser)
         self.dataDir = coreInst.dataDir + '/contacts/'
         self.dataFile = '%s/contacts/%s.json' % (coreInst.dataDir, publicKey)
diff --git a/onionr/onionrusers/onionrusers.py b/onionr/onionrusers/onionrusers.py
index 9e26d047..51977b39 100755
--- a/onionr/onionrusers/onionrusers.py
+++ b/onionr/onionrusers/onionrusers.py
@@ -18,6 +18,7 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 import onionrblockapi, logger, onionrexceptions, json, sqlite3, time
+import unpaddedbase32
 import nacl.exceptions
 
 def deleteExpiredKeys(coreInst):
@@ -55,8 +56,7 @@ class OnionrUser:
             Takes an instance of onionr core, a base32 encoded ed25519 public key, and a bool saveUser
             saveUser determines if we should add a user to our peer database or not.
         '''
-        if ' ' in coreInst._utils.bytesToStr(publicKey).strip():
-            publicKey = coreInst._utils.convertHumanReadableID(publicKey)
+        publicKey = unpaddedbase32.repad(coreInst._utils.strToBytes(publicKey)).decode()
 
         self.trust = 0
         self._core = coreInst
@@ -190,6 +190,7 @@ class OnionrUser:
         return list(keyList)
 
     def addForwardKey(self, newKey, expire=DEFAULT_KEY_EXPIRE):
+        newKey = self._core._utils.bytesToStr(unpaddedbase32.repad(self._core._utils.strToBytes(newKey)))
         if not self._core._utils.validatePubKey(newKey):
             # Do not add if something went wrong with the key
             raise onionrexceptions.InvalidPubkey(newKey)
diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py
index e2a5964d..887c421a 100755
--- a/onionr/onionrutils.py
+++ b/onionr/onionrutils.py
@@ -21,6 +21,7 @@
 import sys, os, sqlite3, binascii, time, base64, json, glob, shutil, math, re, urllib.parse, string
 import requests
 import nacl.signing, nacl.encoding
+import unpaddedbase32
 from onionrblockapi import Block
 import onionrexceptions, config, logger
 from onionr import API_VERSION
@@ -319,9 +320,12 @@ class OnionrUtils:
         '''
             Validate if a string is a valid base32 encoded Ed25519 key
         '''
-        retVal = False
         if type(key) is type(None):
             return False
+        # Accept keys that have no = padding
+        key = unpaddedbase32.repad(self.strToBytes(key))
+
+        retVal = False
         try:
             nacl.signing.SigningKey(seed=key, encoder=nacl.encoding.Base32Encoder)
         except nacl.exceptions.ValueError:
diff --git a/onionr/static-data/www/mail/mail.css b/onionr/static-data/www/mail/mail.css
index 32999440..f2c31086 100755
--- a/onionr/static-data/www/mail/mail.css
+++ b/onionr/static-data/www/mail/mail.css
@@ -113,4 +113,8 @@ input{
     color: black;
     font-size: 1.5em;
     width: 10%;
+}
+
+.content{
+    min-height: 1000px;
 }
\ No newline at end of file
diff --git a/onionr/static-data/www/mail/mail.js b/onionr/static-data/www/mail/mail.js
index cc4895a1..27180173 100755
--- a/onionr/static-data/www/mail/mail.js
+++ b/onionr/static-data/www/mail/mail.js
@@ -58,9 +58,9 @@ function openReply(bHash, quote, subject){
     // Add quoted reply
     var splitQuotes = quote.split('\n')
     for (var x = 0; x < splitQuotes.length; x++){
-        splitQuotes[x] = '>' + splitQuotes[x]
+        splitQuotes[x] = '> ' + splitQuotes[x]
     }
-    quote = '\n' + splitQuotes.join('\n')
+    quote = '\n' + key.substring(0, 12) + ' wrote:' + '\n' + splitQuotes.join('\n')
     document.getElementById('draftText').value = quote
     setActiveTab('send message')
 }
@@ -77,7 +77,7 @@ function openThread(bHash, sender, date, sigBool, pubkey, subjectLine){
     var sigMsg = 'signature'
 
     // show add unknown contact button if peer is unknown but still has pubkey
-    if (sender == pubkey){
+    if (sender === pubkey && sender !== myPub){
         addUnknownContact.style.display = 'inline'
     }
 
diff --git a/onionr/static-data/www/mail/sendmail.js b/onionr/static-data/www/mail/sendmail.js
index f8ff49bc..e2d34c1f 100755
--- a/onionr/static-data/www/mail/sendmail.js
+++ b/onionr/static-data/www/mail/sendmail.js
@@ -54,6 +54,11 @@ sendForm.onsubmit = function(){
             return false
         }
     }
-    sendMail(to.value, messageContent.value, subject.value)
+    if (to.value.length !== 56 && to.value.length !== 52){
+        alert('Public key is not valid')   
+    }
+    else{
+        sendMail(to.value, messageContent.value, subject.value)
+    }
     return false
 }
diff --git a/onionr/static-data/www/private/index.html b/onionr/static-data/www/private/index.html
index 151c3704..a2d3d3c9 100755
--- a/onionr/static-data/www/private/index.html
+++ b/onionr/static-data/www/private/index.html
@@ -21,26 +21,26 @@
             <br><br>
             <div>🕵️‍♂️ Current Used Identity: <input class='myPub' type='text' readonly></div>
             <br>
-            <button id='shutdownNode' class='warnBtn'>Shutdown Node</button> <button id='refreshStats' class='primaryBtn'>Refresh Stats</button>
+            <button id='shutdownNode' class='btn warnBtn'>Shutdown Node</button> <button id='refreshStats' class='btn primaryBtn'>Refresh Stats</button>
             <br><br>
-            <h1>Onionr Services</h1>
             <label>Open Site: <input type='text' id='siteViewer' placeholder='Site Hash'> <button id='openSite' class='primaryBtn openSiteBtn'>Open Onionr Site</button></label>
             <br>
-            <br><br><a class='idLink' href='/mail/'>Mail</a> - <a class='idLink' href='/friends/'>Friend Manager</a> - <a class='idLink' href='/board/'>Boards</a> - 
+            <br><br><a class='idLink' href='/mail/'>Mail</a> - <a class='idLink' href='/friends/'>Friend Manager</a> - <a class='idLink' href='/board/'>Circle</a> - 
                     <a class='idLink' href='/clandestine/'>Clandestine</a>
             <br><br><hr>
             <details class='configArea'>
                 <summary><b>Edit Configuration</b></summary>
                 <br>
-                <p><em>Warning: </em><b>Some values can be dangerous to change. Use caution.</b></p>
+                <p><em>Warning: </em><b>Some values can be dangerous to change.<br><br>Configuration contains sensitive information.</b></p>
                 <br>
                 <textarea class='configEditor'></textarea>
                 <button class='saveConfig successBtn'>Save Config</button>
             </details>
             <hr>
-            <h1>Statistics</h1>
+            <p>🔒 Security Level: <span id='securityLevel'></span></p>
             <p>🕰️ Uptime: <span id='uptime'></span></p>
             <h2>Connections</h2>
+            <p class='secRequestNotice hidden'>Note: on high security levels, you should have <em>no</em> received requests.</p>
             <p>🖇️ Last Received Request: <span id='lastIncoming'>None since start</span></p>
             <p>⬇️ Total Requests Received: <span id='totalRec'>None since start</span></p>
             <p>🔗 Outgoing Connections:</p>
diff --git a/onionr/static-data/www/private/main.css b/onionr/static-data/www/private/main.css
index 91a7b588..896cd2f5 100755
--- a/onionr/static-data/www/private/main.css
+++ b/onionr/static-data/www/private/main.css
@@ -5,4 +5,14 @@
 
 .saveConfig{
     margin-top: 1em;
+}
+
+
+.idLink{
+    -webkit-touch-callout: none; /* iOS Safari */
+    -webkit-user-select: none; /* Safari */
+       -moz-user-select: none; /* Firefox */
+        -ms-user-select: none; /* Internet Explorer/Edge */
+            user-select: none; /* Non-prefixed version, currently
+                                  supported by Chrome and Opera */
 }
\ No newline at end of file
diff --git a/onionr/static-data/www/shared/main/stats.js b/onionr/static-data/www/shared/main/stats.js
index 86b7f451..9b50185e 100755
--- a/onionr/static-data/www/shared/main/stats.js
+++ b/onionr/static-data/www/shared/main/stats.js
@@ -18,10 +18,34 @@
 */
 uptimeDisplay = document.getElementById('uptime')
 connectedDisplay = document.getElementById('connectedNodes')
+connectedDisplay.style.maxHeight = '300px'
+connectedDisplay.style.overflowY = 'scroll'
 storedBlockDisplay = document.getElementById('storedBlocks')
 queuedBlockDisplay = document.getElementById('blockQueue')
 lastIncoming = document.getElementById('lastIncoming')
 totalRec = document.getElementById('totalRec')
+securityLevel = document.getElementById('securityLevel')
+sec_description_str = 'unknown'
+
+function showSecStatNotice(){
+    var secWarnEls = document.getElementsByClassName('secRequestNotice')
+    for (el = 0; el < secWarnEls.length; el++){
+        secWarnEls[el].style.display = 'block'
+    }
+}
+
+switch (httpGet('/config/get/general.security_level')){
+    case "0":
+        sec_description_str = 'normal'
+        break;
+    case "1":
+        sec_description_str = 'high'
+        break;
+}
+
+if (sec_description_str !== 'normal'){
+    showSecStatNotice()
+}
 
 function getStats(){
     stats = JSON.parse(httpGet('getstats', webpass))
@@ -29,6 +53,7 @@ function getStats(){
     connectedDisplay.innerText = stats['connectedNodes']
     storedBlockDisplay.innerText = stats['blockCount']
     queuedBlockDisplay.innerText = stats['blockQueueCount']
+    securityLevel.innerText = sec_description_str
     totalRec.innerText = httpGet('/hitcount')
     var lastConnect = httpGet('/lastconnect')
     if (lastConnect > 0){
diff --git a/onionr/static-data/www/shared/main/style.css b/onionr/static-data/www/shared/main/style.css
index 18388845..8181df8d 100755
--- a/onionr/static-data/www/shared/main/style.css
+++ b/onionr/static-data/www/shared/main/style.css
@@ -178,8 +178,16 @@ body{
     background-color:#396BAC;    
 }
 
+.btn:hover{
+    opacity: 0.6;
+}
+
 .openSiteBtn{
     padding: 5px;
     border: 1px solid black;
     border-radius: 5px;
 }
+
+.hidden{
+    display: none;
+}
diff --git a/onionr/tests/test_stringvalidations.py b/onionr/tests/test_stringvalidations.py
index 0cc45eae..caac34df 100755
--- a/onionr/tests/test_stringvalidations.py
+++ b/onionr/tests/test_stringvalidations.py
@@ -29,11 +29,13 @@ class OnionrValidations(unittest.TestCase):
     
     def test_pubkey_validator(self):
         # Test ed25519 public key validity
-        valid = 'JZ5VE72GUS3C7BOHDRIYZX4B5U5EJMCMLKHLYCVBQQF3UKHYIRRQ===='
+        valids = ['JZ5VE72GUS3C7BOHDRIYZX4B5U5EJMCMLKHLYCVBQQF3UKHYIRRQ====', 'JZ5VE72GUS3C7BOHDRIYZX4B5U5EJMCMLKHLYCVBQQF3UKHYIRRQ']
         invalid = [None, '', '    ', 'dfsg', '\n', 'JZ5VE72GUS3C7BOHDRIYZX4B5U5EJMCMLKHLYCVBQQF3UKHYIR$Q====']
         c = core.Core()
-        print('testing', valid)
-        self.assertTrue(c._utils.validatePubKey(valid))
+
+        for valid in valids:
+            print('testing', valid)
+            self.assertTrue(c._utils.validatePubKey(valid))
 
         for x in invalid:
             #print('testing', x)
diff --git a/requirements.in b/requirements.in
index bbb9cdc3..c285834a 100644
--- a/requirements.in
+++ b/requirements.in
@@ -1,9 +1,10 @@
 urllib3==1.24.2
 requests==2.21.0
-PyNaCl==1.2.1
+PyNaCl==1.3.0
 gevent==1.3.6
 Flask==1.0.2
 PySocks==1.6.8
 stem==1.7.1
 deadsimplekv==0.1.1
+unpaddedbase32==0.1.0
 jinja2==2.10.1
diff --git a/requirements.txt b/requirements.txt
index 06b3303d..261a9996 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --generate-hashes --output-file requirements.txt requirements.in
+#    pip-compile --generate-hashes --output-file=requirements.txt requirements.in
 #
 certifi==2018.11.29 \
     --hash=sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7 \
@@ -140,38 +140,26 @@ markupsafe==1.1.1 \
 pycparser==2.19 \
     --hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3 \
     # via cffi
-pynacl==1.2.1 \
-    --hash=sha256:04e30e5bdeeb2d5b34107f28cd2f5bbfdc6c616f3be88fc6f53582ff1669eeca \
-    --hash=sha256:0bfa0d94d2be6874e40f896e0a67e290749151e7de767c5aefbad1121cad7512 \
-    --hash=sha256:11aa4e141b2456ce5cecc19c130e970793fa3a2c2e6fbb8ad65b28f35aa9e6b6 \
-    --hash=sha256:13bdc1fe084ff9ac7653ae5a924cae03bf4bb07c6667c9eb5b6eb3c570220776 \
-    --hash=sha256:14339dc233e7a9dda80a3800e64e7ff89d0878ba23360eea24f1af1b13772cac \
-    --hash=sha256:1d33e775fab3f383167afb20b9927aaf4961b953d76eeb271a5703a6d756b65b \
-    --hash=sha256:2a42b2399d0428619e58dac7734838102d35f6dcdee149e0088823629bf99fbb \
-    --hash=sha256:2dce05ac8b3c37b9e2f65eab56c544885607394753e9613fd159d5e2045c2d98 \
-    --hash=sha256:63cfccdc6217edcaa48369191ae4dca0c390af3c74f23c619e954973035948cd \
-    --hash=sha256:6453b0dae593163ffc6db6f9c9c1597d35c650598e2c39c0590d1757207a1ac2 \
-    --hash=sha256:73a5a96fb5fbf2215beee2353a128d382dbca83f5341f0d3c750877a236569ef \
-    --hash=sha256:8abb4ef79161a5f58848b30ab6fb98d8c466da21fdd65558ce1d7afc02c70b5f \
-    --hash=sha256:8ac1167195b32a8755de06efd5b2d2fe76fc864517dab66aaf65662cc59e1988 \
-    --hash=sha256:8f505f42f659012794414fa57c498404e64db78f1d98dfd40e318c569f3c783b \
-    --hash=sha256:9c8a06556918ee8e3ab48c65574f318f5a0a4d31437fc135da7ee9d4f9080415 \
-    --hash=sha256:a1e25fc5650cf64f01c9e435033e53a4aca9de30eb9929d099f3bb078e18f8f2 \
-    --hash=sha256:be71cd5fce04061e1f3d39597f93619c80cdd3558a6c9ba99a546f144a8d8101 \
-    --hash=sha256:c5b1a7a680218dee9da0f1b5e24072c46b3c275d35712bc1d505b85bb03441c0 \
-    --hash=sha256:cb785db1a9468841a1265c9215c60fe5d7af2fb1b209e3316a152704607fc582 \
-    --hash=sha256:cf6877124ae6a0698404e169b3ba534542cfbc43f939d46b927d956daf0a373a \
-    --hash=sha256:d0eb5b2795b7ee2cbcfcadacbe95a13afbda048a262bd369da9904fecb568975 \
-    --hash=sha256:d3a934e2b9f20abac009d5b6951067cfb5486889cb913192b4d8288b216842f1 \
-    --hash=sha256:d795f506bcc9463efb5ebb0f65ed77921dcc9e0a50499dedd89f208445de9ecb \
-    --hash=sha256:d8aaf7e5d6b0e0ef7d6dbf7abeb75085713d0100b4eb1a4e4e857de76d77ac45 \
-    --hash=sha256:de2aaca8386cf4d70f1796352f2346f48ddb0bed61dc43a3ce773ba12e064031 \
-    --hash=sha256:e0d38fa0a75f65f556fb912f2c6790d1fa29b7dd27a1d9cc5591b281321eaaa9 \
-    --hash=sha256:eb2acabbd487a46b38540a819ef67e477a674481f84a82a7ba2234b9ba46f752 \
-    --hash=sha256:eeee629828d0eb4f6d98ac41e9a3a6461d114d1d0aa111a8931c049359298da0 \
-    --hash=sha256:f5836463a3c0cca300295b229b6c7003c415a9d11f8f9288ddbd728e2746524c \
-    --hash=sha256:f5ce9e26d25eb0b2d96f3ef0ad70e1d3ae89b5d60255c462252a3e456a48c053 \
-    --hash=sha256:fabf73d5d0286f9e078774f3435601d2735c94ce9e514ac4fb945701edead7e4
+pynacl==1.3.0 \
+    --hash=sha256:05c26f93964373fc0abe332676cb6735f0ecad27711035b9472751faa8521255 \
+    --hash=sha256:0c6100edd16fefd1557da078c7a31e7b7d7a52ce39fdca2bec29d4f7b6e7600c \
+    --hash=sha256:0d0a8171a68edf51add1e73d2159c4bc19fc0718e79dec51166e940856c2f28e \
+    --hash=sha256:1c780712b206317a746ace34c209b8c29dbfd841dfbc02aa27f2084dd3db77ae \
+    --hash=sha256:2424c8b9f41aa65bbdbd7a64e73a7450ebb4aa9ddedc6a081e7afcc4c97f7621 \
+    --hash=sha256:2d23c04e8d709444220557ae48ed01f3f1086439f12dbf11976e849a4926db56 \
+    --hash=sha256:30f36a9c70450c7878053fa1344aca0145fd47d845270b43a7ee9192a051bf39 \
+    --hash=sha256:37aa336a317209f1bb099ad177fef0da45be36a2aa664507c5d72015f956c310 \
+    --hash=sha256:4943decfc5b905748f0756fdd99d4f9498d7064815c4cf3643820c9028b711d1 \
+    --hash=sha256:57ef38a65056e7800859e5ba9e6091053cd06e1038983016effaffe0efcd594a \
+    --hash=sha256:5bd61e9b44c543016ce1f6aef48606280e45f892a928ca7068fba30021e9b786 \
+    --hash=sha256:6482d3017a0c0327a49dddc8bd1074cc730d45db2ccb09c3bac1f8f32d1eb61b \
+    --hash=sha256:7d3ce02c0784b7cbcc771a2da6ea51f87e8716004512493a2b69016326301c3b \
+    --hash=sha256:a14e499c0f5955dcc3991f785f3f8e2130ed504fa3a7f44009ff458ad6bdd17f \
+    --hash=sha256:a39f54ccbcd2757d1d63b0ec00a00980c0b382c62865b61a505163943624ab20 \
+    --hash=sha256:aabb0c5232910a20eec8563503c153a8e78bbf5459490c49ab31f6adf3f3a415 \
+    --hash=sha256:bd4ecb473a96ad0f90c20acba4f0bf0df91a4e03a1f4dd6a4bdc9ca75aa3a715 \
+    --hash=sha256:e2da3c13307eac601f3de04887624939aca8ee3c9488a0bb0eca4fb9401fc6b1 \
+    --hash=sha256:f67814c38162f4deb31f68d590771a29d5ae3b1bd64b75cf232308e5c74777e0
 pysocks==1.6.8 \
     --hash=sha256:3fe52c55890a248676fd69dc9e3c4e811718b777834bcaab7a8125cf9deac672
 requests==2.21.0 \
@@ -183,6 +171,8 @@ six==1.12.0 \
     # via pynacl
 stem==1.7.1 \
     --hash=sha256:c9eaf3116cb60c15995cbd3dec3a5cbc50e9bb6e062c4d6d42201e566f498ca2
+unpaddedbase32==0.1.0 \
+    --hash=sha256:5e4143fcaf77c9c6b4f60d18301c7570f0dac561dcf9b9aed8b5ba6ead7f218c
 urllib3==1.24.2 \
     --hash=sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0 \
     --hash=sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3

From 065e97ab118718515715ee22aa3038fb86d778b3 Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Wed, 19 Jun 2019 15:29:27 -0500
Subject: [PATCH 02/29] improved logging messages to be less spammy

---
 README.md                                     |   2 +-
 onionr/communicator.py                        |  16 +--
 onionr/communicatorutils/connectnewpeers.py   |   2 +-
 onionr/communicatorutils/downloadblocks.py    |   1 +
 onionr/communicatorutils/lookupblocks.py      | 116 +++++++++---------
 onionr/logger.py                              |  32 ++---
 onionr/netcontroller.py                       |  11 +-
 onionr/onionr.py                              |  10 +-
 onionr/onionrcommands/banblocks.py            |   8 +-
 onionr/onionrcommands/daemonlaunch.py         |  32 ++---
 onionr/onionrcommands/onionrstatistics.py     |  16 +--
 onionr/onionrcommands/openwebinterface.py     |   2 +-
 onionr/onionrutils.py                         |   2 +-
 .../static-data/default-plugins/flow/main.py  |   8 +-
 14 files changed, 129 insertions(+), 129 deletions(-)

diff --git a/README.md b/README.md
index 536e323b..4417e31f 100644
--- a/README.md
+++ b/README.md
@@ -85,7 +85,7 @@ The following applies to Ubuntu Bionic. Other distros may have different package
 
 `$ sudo apt install python3-pip python3-dev tor`
 
-* Have python3.6+, python3-pip, Tor (daemon, not browser) installed (python3-dev recommended)
+* Have python3.6+, python3-pip, Tor (daemon, not browser) installed. python3-dev is recommended.
 * Clone the git repo: `$ git clone https://gitlab.com/beardog/onionr`
 * cd into install direction: `$ cd onionr/`
 * Install the Python dependencies ([virtualenv strongly recommended](https://virtualenv.pypa.io/en/stable/userguide/)): `$ pip3 install --require-hashes -r requirements.txt`
diff --git a/onionr/communicator.py b/onionr/communicator.py
index de49e601..370a4981 100755
--- a/onionr/communicator.py
+++ b/onionr/communicator.py
@@ -176,7 +176,7 @@ class OnionrCommunicatorDaemon:
             self.shutdown = True
             pass
 
-        logger.info('Goodbye. (Onionr is cleaning up, and will exit)')
+        logger.info('Goodbye. (Onionr is cleaning up, and will exit)', terminal=True)
         try:
             self.service_greenlets
         except AttributeError:
@@ -252,7 +252,7 @@ class OnionrCommunicatorDaemon:
                 break
         else:
             if len(self.onlinePeers) == 0:
-                logger.debug('Couldn\'t connect to any peers.' + (' Last node seen %s ago.' % humanreadabletime.human_readable_time(time.time() - self.lastNodeSeen) if not self.lastNodeSeen is None else ''))
+                logger.debug('Couldn\'t connect to any peers.' + (' Last node seen %s ago.' % humanreadabletime.human_readable_time(time.time() - self.lastNodeSeen) if not self.lastNodeSeen is None else ''), terminal=True)
             else:
                 self.lastNodeSeen = time.time()
         self.decrementThreadCount('getOnlinePeers')
@@ -293,12 +293,12 @@ class OnionrCommunicatorDaemon:
     def printOnlinePeers(self):
         '''logs online peer list'''
         if len(self.onlinePeers) == 0:
-            logger.warn('No online peers')
+            logger.warn('No online peers', terminal=True)
         else:
-            logger.info('Online peers:')
+            logger.info('Online peers:', terminal=True)
             for i in self.onlinePeers:
                 score = str(self.getPeerProfileInstance(i).score)
-                logger.info(i + ', score: ' + score)
+                logger.info(i + ', score: ' + score, terminal=True)
 
     def peerAction(self, peer, action, data='', returnHeaders=False):
         '''Perform a get request to a peer'''
@@ -318,6 +318,7 @@ class OnionrCommunicatorDaemon:
                 self.getPeerProfileInstance(peer).addScore(-10)
                 self.removeOnlinePeer(peer)
                 if action != 'ping':
+                    logger.warn('Lost connection to ' + peer, terminal=True)
                     self.getOnlinePeers() # Will only add a new peer to pool if needed
             except ValueError:
                 pass
@@ -359,7 +360,7 @@ class OnionrCommunicatorDaemon:
     def announce(self, peer):
         '''Announce to peers our address'''
         if announcenode.announce_node(self) == False:
-            logger.warn('Could not introduce node.')
+            logger.warn('Could not introduce node.', terminal=True)
 
     def detectAPICrash(self):
         '''exit if the api server crashes/stops'''
@@ -371,7 +372,7 @@ class OnionrCommunicatorDaemon:
             else:
                 # This executes if the api is NOT detected to be running
                 events.event('daemon_crash', onionr = self._core.onionrInst, data = {})
-                logger.error('Daemon detected API crash (or otherwise unable to reach API after long time), stopping...')
+                logger.fatal('Daemon detected API crash (or otherwise unable to reach API after long time), stopping...', terminal=True)
                 self.shutdown = True
         self.decrementThreadCount('detectAPICrash')
 
@@ -388,5 +389,4 @@ def run_file_exists(daemon):
     if os.path.isfile(daemon._core.dataDir + '.runcheck'):
         os.remove(daemon._core.dataDir + '.runcheck')
         return True
-
     return False
\ No newline at end of file
diff --git a/onionr/communicatorutils/connectnewpeers.py b/onionr/communicatorutils/connectnewpeers.py
index ee1eb468..96e4ecac 100755
--- a/onionr/communicatorutils/connectnewpeers.py
+++ b/onionr/communicatorutils/connectnewpeers.py
@@ -72,7 +72,7 @@ def connect_new_peer_to_communicator(comm_inst, peer='', useBootstrap=False):
                 # Add a peer to our list if it isn't already since it successfully connected
                 networkmerger.mergeAdders(address, comm_inst._core)
             if address not in comm_inst.onlinePeers:
-                logger.info('Connected to ' + address)
+                logger.info('Connected to ' + address, terminal=True)
                 comm_inst.onlinePeers.append(address)
                 comm_inst.connectTimes[address] = comm_inst._core._utils.getEpoch()
             retData = address
diff --git a/onionr/communicatorutils/downloadblocks.py b/onionr/communicatorutils/downloadblocks.py
index 720e0eae..aad5f884 100755
--- a/onionr/communicatorutils/downloadblocks.py
+++ b/onionr/communicatorutils/downloadblocks.py
@@ -110,6 +110,7 @@ def download_blocks_from_communicator(comm_inst):
             if removeFromQueue:
                 try:
                     del comm_inst.blockQueue[blockHash] # remove from block queue both if success or false
+                    logger.info('%s blocks remaining in queue' % [len(comm_inst.blockQueue)])
                 except KeyError:
                     pass
         comm_inst.currentDownloading.remove(blockHash)
diff --git a/onionr/communicatorutils/lookupblocks.py b/onionr/communicatorutils/lookupblocks.py
index 0994d15c..e39e3c68 100755
--- a/onionr/communicatorutils/lookupblocks.py
+++ b/onionr/communicatorutils/lookupblocks.py
@@ -19,61 +19,65 @@
 '''
 import logger, onionrproofs
 def lookup_blocks_from_communicator(comm_inst):
-        logger.info('Looking up new blocks...')
-        tryAmount = 2
-        newBlocks = ''
-        existingBlocks = comm_inst._core.getBlockList()
-        triedPeers = [] # list of peers we've tried this time around
-        maxBacklog = 1560 # Max amount of *new* block hashes to have already in queue, to avoid memory exhaustion
-        lastLookupTime = 0 # Last time we looked up a particular peer's list
-        for i in range(tryAmount):
-            listLookupCommand = 'getblocklist' # This is defined here to reset it each time
-            if len(comm_inst.blockQueue) >= maxBacklog:
+    logger.info('Looking up new blocks...')
+    tryAmount = 2
+    newBlocks = ''
+    existingBlocks = comm_inst._core.getBlockList()
+    triedPeers = [] # list of peers we've tried this time around
+    maxBacklog = 1560 # Max amount of *new* block hashes to have already in queue, to avoid memory exhaustion
+    lastLookupTime = 0 # Last time we looked up a particular peer's list
+    new_block_count = 0
+    for i in range(tryAmount):
+        listLookupCommand = 'getblocklist' # This is defined here to reset it each time
+        if len(comm_inst.blockQueue) >= maxBacklog:
+            break
+        if not comm_inst.isOnline:
+            break
+        # check if disk allocation is used
+        if comm_inst._core._utils.storageCounter.isFull():
+            logger.debug('Not looking up new blocks due to maximum amount of allowed disk space used')
+            break
+        peer = comm_inst.pickOnlinePeer() # select random online peer
+        # if we've already tried all the online peers this time around, stop
+        if peer in triedPeers:
+            if len(comm_inst.onlinePeers) == len(triedPeers):
                 break
-            if not comm_inst.isOnline:
-                break
-            # check if disk allocation is used
-            if comm_inst._core._utils.storageCounter.isFull():
-                logger.debug('Not looking up new blocks due to maximum amount of allowed disk space used')
-                break
-            peer = comm_inst.pickOnlinePeer() # select random online peer
-            # if we've already tried all the online peers this time around, stop
-            if peer in triedPeers:
-                if len(comm_inst.onlinePeers) == len(triedPeers):
-                    break
-                else:
-                    continue
-            triedPeers.append(peer)
+            else:
+                continue
+        triedPeers.append(peer)
 
-            # Get the last time we looked up a peer's stamp to only fetch blocks since then.
-            # Saved in memory only for privacy reasons
-            try:
-                lastLookupTime = comm_inst.dbTimestamps[peer]
-            except KeyError:
-                lastLookupTime = 0
-            else:
-                listLookupCommand += '?date=%s' % (lastLookupTime,)
-            try:
-                newBlocks = comm_inst.peerAction(peer, listLookupCommand) # get list of new block hashes
-            except Exception as error:
-                logger.warn('Could not get new blocks from %s.' % peer, error = error)
-                newBlocks = False
-            else:
-                comm_inst.dbTimestamps[peer] = comm_inst._core._utils.getRoundedEpoch(roundS=60)
-            if newBlocks != False:
-                # if request was a success
-                for i in newBlocks.split('\n'):
-                    if comm_inst._core._utils.validateHash(i):
-                        # if newline seperated string is valid hash
-                        if not i in existingBlocks:
-                            # if block does not exist on disk and is not already in block queue
-                            if i not in comm_inst.blockQueue:
-                                if onionrproofs.hashMeetsDifficulty(i) and not comm_inst._core._blacklist.inBlacklist(i):
-                                    if len(comm_inst.blockQueue) <= 1000000:
-                                        comm_inst.blockQueue[i] = [peer] # add blocks to download queue
-                            else:
-                                if peer not in comm_inst.blockQueue[i]:
-                                    if len(comm_inst.blockQueue[i]) < 10:
-                                        comm_inst.blockQueue[i].append(peer)
-        comm_inst.decrementThreadCount('lookupBlocks')
-        return
\ No newline at end of file
+        # Get the last time we looked up a peer's stamp to only fetch blocks since then.
+        # Saved in memory only for privacy reasons
+        try:
+            lastLookupTime = comm_inst.dbTimestamps[peer]
+        except KeyError:
+            lastLookupTime = 0
+        else:
+            listLookupCommand += '?date=%s' % (lastLookupTime,)
+        try:
+            newBlocks = comm_inst.peerAction(peer, listLookupCommand) # get list of new block hashes
+        except Exception as error:
+            logger.warn('Could not get new blocks from %s.' % peer, error = error)
+            newBlocks = False
+        else:
+            comm_inst.dbTimestamps[peer] = comm_inst._core._utils.getRoundedEpoch(roundS=60)
+        if newBlocks != False:
+            # if request was a success
+            for i in newBlocks.split('\n'):
+                if comm_inst._core._utils.validateHash(i):
+                    # if newline seperated string is valid hash
+                    if not i in existingBlocks:
+                        # if block does not exist on disk and is not already in block queue
+                        if i not in comm_inst.blockQueue:
+                            if onionrproofs.hashMeetsDifficulty(i) and not comm_inst._core._blacklist.inBlacklist(i):
+                                if len(comm_inst.blockQueue) <= 1000000:
+                                    comm_inst.blockQueue[i] = [peer] # add blocks to download queue
+                                    new_block_count += 1
+                        else:
+                            if peer not in comm_inst.blockQueue[i]:
+                                if len(comm_inst.blockQueue[i]) < 10:
+                                    comm_inst.blockQueue[i].append(peer)
+    if new_block_count > 0:
+        logger.info('Discovered %s new blocks' % (new_block_count,), terminal=True)
+    comm_inst.decrementThreadCount('lookupBlocks')
+    return
\ No newline at end of file
diff --git a/onionr/logger.py b/onionr/logger.py
index a7abf715..01ada8ce 100755
--- a/onionr/logger.py
+++ b/onionr/logger.py
@@ -126,24 +126,24 @@ def get_file():
 
     return _outputfile
 
-def raw(data, fd = sys.stdout, sensitive = False):
+def raw(data, fd = sys.stdout, terminal = False):
     '''
         Outputs raw data to console without formatting
     '''
 
-    if get_settings() & OUTPUT_TO_CONSOLE:
+    if terminal and (get_settings() & OUTPUT_TO_CONSOLE):
         try:
             ts = fd.write('%s\n' % data)
         except OSError:
             pass
-    if get_settings() & OUTPUT_TO_FILE and not sensitive:
+    if get_settings() & OUTPUT_TO_FILE:
         try:
             with open(_outputfile, "a+") as f:
                 f.write(colors.filter(data) + '\n')
         except OSError:
             pass
 
-def log(prefix, data, color = '', timestamp=True, fd = sys.stdout, prompt = True, sensitive = False):
+def log(prefix, data, color = '', timestamp=True, fd = sys.stdout, prompt = True, terminal = False):
     '''
         Logs the data
         prefix : The prefix to the output
@@ -158,7 +158,7 @@ def log(prefix, data, color = '', timestamp=True, fd = sys.stdout, prompt = True
     if not get_settings() & USE_ANSI:
         output = colors.filter(output)
 
-    raw(output, fd = fd, sensitive = sensitive)
+    raw(output, fd = fd, terminal = terminal)
 
 def readline(message = ''):
     '''
@@ -210,37 +210,37 @@ def confirm(default = 'y', message = 'Are you sure %s? '):
         return default == 'y'
 
 # debug: when there is info that could be useful for debugging purposes only
-def debug(data, error = None, timestamp = True, prompt = True, sensitive = False, level = LEVEL_DEBUG):
+def debug(data, error = None, timestamp = True, prompt = True, terminal = False, level = LEVEL_DEBUG):
     if get_level() <= level:
-        log('/', data, timestamp = timestamp, prompt = prompt, sensitive = sensitive)
+        log('/', data, timestamp = timestamp, prompt = prompt, terminal = terminal)
     if not error is None:
         debug('Error: ' + str(error) + parse_error())
 
 # info: when there is something to notify the user of, such as the success of a process
-def info(data, timestamp = False, prompt = True, sensitive = False, level = LEVEL_INFO):
+def info(data, timestamp = False, prompt = True, terminal = False, level = LEVEL_INFO):
     if get_level() <= level:
-        log('+', data, colors.fg.green, timestamp = timestamp, prompt = prompt, sensitive = sensitive)
+        log('+', data, colors.fg.green, timestamp = timestamp, prompt = prompt, terminal = terminal)
 
 # warn: when there is a potential for something bad to happen
-def warn(data, error = None, timestamp = True, prompt = True, sensitive = False, level = LEVEL_WARN):
+def warn(data, error = None, timestamp = True, prompt = True, terminal = False, level = LEVEL_WARN):
     if not error is None:
         debug('Error: ' + str(error) + parse_error())
     if get_level() <= level:
-        log('!', data, colors.fg.orange, timestamp = timestamp, prompt = prompt, sensitive = sensitive)
+        log('!', data, colors.fg.orange, timestamp = timestamp, prompt = prompt, terminal = terminal)
 
 # error: when only one function, module, or process of the program encountered a problem and must stop
-def error(data, error = None, timestamp = True, prompt = True, sensitive = False, level = LEVEL_ERROR):
+def error(data, error = None, timestamp = True, prompt = True, terminal = False, level = LEVEL_ERROR):
     if get_level() <= level:
-        log('-', data, colors.fg.red, timestamp = timestamp, fd = sys.stderr, prompt = prompt, sensitive = sensitive)
+        log('-', data, colors.fg.red, timestamp = timestamp, fd = sys.stderr, prompt = prompt, terminal = terminal)
     if not error is None:
         debug('Error: ' + str(error) + parse_error())
 
 # fatal: when the something so bad has happened that the program must stop
-def fatal(data, error = None, timestamp=True, prompt = True, sensitive = False, level = LEVEL_FATAL):
+def fatal(data, error = None, timestamp=True, prompt = True, terminal = False, level = LEVEL_FATAL):
     if not error is None:
-        debug('Error: ' + str(error) + parse_error(), sensitive = sensitive)
+        debug('Error: ' + str(error) + parse_error(), terminal = terminal)
     if get_level() <= level:
-        log('#', data, colors.bg.red + colors.fg.green + colors.bold, timestamp = timestamp, fd = sys.stderr, prompt = prompt, sensitive = sensitive)
+        log('#', data, colors.bg.red + colors.fg.green + colors.bold, timestamp = timestamp, fd = sys.stderr, prompt = prompt, terminal = terminal)
 
 # returns a formatted error message
 def parse_error():
diff --git a/onionr/netcontroller.py b/onionr/netcontroller.py
index 74087500..b2d8237f 100755
--- a/onionr/netcontroller.py
+++ b/onionr/netcontroller.py
@@ -124,14 +124,14 @@ HiddenServicePort 80 ''' + self.apiServerIP + ''':''' + str(self.hsPort)
         try:
             tor = subprocess.Popen([self.torBinary, '-f', self.torConfigLocation], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         except FileNotFoundError:
-            logger.fatal("Tor was not found in your path or the Onionr directory. Please install Tor and try again.")
+            logger.fatal("Tor was not found in your path or the Onionr directory. Please install Tor and try again.", terminal=True)
             sys.exit(1)
         else:
             # Test Tor Version
             torVersion = subprocess.Popen([self.torBinary, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
             for line in iter(torVersion.stdout.readline, b''):
                 if 'Tor 0.2.' in line.decode():
-                    logger.error('Tor 0.3+ required')
+                    logger.fatal('Tor 0.3+ required', terminal=True)
                     sys.exit(1)
                     break
             torVersion.kill()
@@ -140,17 +140,18 @@ HiddenServicePort 80 ''' + self.apiServerIP + ''':''' + str(self.hsPort)
         try:
             for line in iter(tor.stdout.readline, b''):
                 if 'bootstrapped 100' in line.decode().lower():
+                    logger.info(line.decode())
                     break
                 elif 'opening socks listener' in line.decode().lower():
                     logger.debug(line.decode().replace('\n', ''))
             else:
-                logger.fatal('Failed to start Tor. Maybe a stray instance of Tor used by Onionr is still running? This can also be a result of file permissions being too open')
+                logger.fatal('Failed to start Tor. Maybe a stray instance of Tor used by Onionr is still running? This can also be a result of file permissions being too open', terminal=True)
                 return False
         except KeyboardInterrupt:
-            logger.fatal('Got keyboard interrupt. Onionr will exit soon.', timestamp = False, level = logger.LEVEL_IMPORTANT)
+            logger.fatal('Got keyboard interrupt. Onionr will exit soon.', timestamp = False, level = logger.LEVEL_IMPORTANT, terminal=True)
             return False
 
-        logger.debug('Finished starting Tor.', timestamp=True)
+        logger.info('Finished starting Tor.', terminal=True)
         self.readyState = True
 
         try:
diff --git a/onionr/onionr.py b/onionr/onionr.py
index 16ec1839..6306acca 100755
--- a/onionr/onionr.py
+++ b/onionr/onionr.py
@@ -159,7 +159,7 @@ class Onionr:
                 sys.stderr.write(file.read().decode().replace('P', logger.colors.fg.pink).replace('W', logger.colors.reset + logger.colors.bold).replace('G', logger.colors.fg.green).replace('\n', logger.colors.reset + '\n').replace('B', logger.colors.bold).replace('A', '%s' % API_VERSION).replace('V', ONIONR_VERSION))
 
                 if not message is None:
-                    logger.info(logger.colors.fg.lightgreen + '-> ' + str(message) + logger.colors.reset + logger.colors.fg.lightgreen + ' <-\n', sensitive=True)
+                    logger.info(logger.colors.fg.lightgreen + '-> ' + str(message) + logger.colors.reset + logger.colors.fg.lightgreen + ' <-\n', terminal=True)
 
     def deleteRunFiles(self):
         try:
@@ -238,7 +238,7 @@ class Onionr:
         return config.get('client.webpassword')
 
     def printWebPassword(self):
-        logger.info(self.getWebPassword(), sensitive = True)
+        logger.info(self.getWebPassword(), term_only = True)
 
     def getHelp(self):
         return self.cmdhelp
@@ -289,11 +289,11 @@ class Onionr:
             Displays the Onionr version
         '''
 
-        function('Onionr v%s (%s) (API v%s)' % (ONIONR_VERSION, platform.machine(), API_VERSION))
+        function('Onionr v%s (%s) (API v%s)' % (ONIONR_VERSION, platform.machine(), API_VERSION), terminal=True)
         if verbosity >= 1:
-            function(ONIONR_TAGLINE)
+            function(ONIONR_TAGLINE, terminal=True)
         if verbosity >= 2:
-            function('Running on %s %s' % (platform.platform(), platform.release()))
+            function('Running on %s %s' % (platform.platform(), platform.release()), terminal=True)
 
     def doPEX(self):
         '''make communicator do pex'''
diff --git a/onionr/onionrcommands/banblocks.py b/onionr/onionrcommands/banblocks.py
index a9caa867..fe50d16a 100755
--- a/onionr/onionrcommands/banblocks.py
+++ b/onionr/onionrcommands/banblocks.py
@@ -30,10 +30,10 @@ def ban_block(o_inst):
                 o_inst.onionrCore._blacklist.addToDB(ban)
                 o_inst.onionrCore.removeBlock(ban)
             except Exception as error:
-                logger.error('Could not blacklist block', error=error)
+                logger.error('Could not blacklist block', error=error, terminal=True)
             else:
-                logger.info('Block blacklisted')
+                logger.info('Block blacklisted', terminal=True)
         else:
-            logger.warn('That block is already blacklisted')
+            logger.warn('That block is already blacklisted', terminal=True)
     else:
-        logger.error('Invalid block hash')
\ No newline at end of file
+        logger.error('Invalid block hash', terminal=True)
\ No newline at end of file
diff --git a/onionr/onionrcommands/daemonlaunch.py b/onionr/onionrcommands/daemonlaunch.py
index 88e43fae..d65a33d6 100755
--- a/onionr/onionrcommands/daemonlaunch.py
+++ b/onionr/onionrcommands/daemonlaunch.py
@@ -40,11 +40,6 @@ def daemon(o_inst):
 
     Thread(target=api.API, args=(o_inst, o_inst.debug, onionr.API_VERSION)).start()
     Thread(target=api.PublicAPI, args=[o_inst.getClientApi()]).start()
-    try:
-        time.sleep(0)
-    except KeyboardInterrupt:
-        logger.debug('Got keyboard interrupt, shutting down...')
-        _proper_shutdown(o_inst)
 
     apiHost = ''
     while apiHost == '':
@@ -56,10 +51,17 @@ def daemon(o_inst):
         time.sleep(0.5)
     #onionr.Onionr.setupConfig('data/', self = o_inst)
 
+    logger.raw('', terminal=True)
+    # print nice header thing :)
+    if o_inst.onionrCore.config.get('general.display_header', True):
+        o_inst.header()
+    o_inst.version(verbosity = 5, function = logger.info)
+    logger.debug('Python version %s' % platform.python_version())
+
     if o_inst._developmentMode:
-        logger.warn('DEVELOPMENT MODE ENABLED (NOT RECOMMENDED)', timestamp = False)
+        logger.warn('DEVELOPMENT MODE ENABLED', timestamp = False, terminal=True)
     net = NetController(o_inst.onionrCore.config.get('client.public.port', 59497), apiServerIP=apiHost)
-    logger.debug('Tor is starting...')
+    logger.info('Tor is starting...', terminal=True)
     if not net.startTor():
         o_inst.onionrUtils.localCommand('shutdown')
         sys.exit(1)
@@ -67,7 +69,7 @@ def daemon(o_inst):
         logger.debug('Started .onion service: %s' % (logger.colors.underline + net.myID))
     else:
         logger.debug('.onion service disabled')
-    logger.debug('Using public key: %s' % (logger.colors.underline + o_inst.onionrCore._crypto.pubKey))
+    logger.info('Using public key: %s' % (logger.colors.underline + o_inst.onionrCore._crypto.pubKey[:52]), terminal=True)
 
     try:
         time.sleep(1)
@@ -81,14 +83,6 @@ def daemon(o_inst):
     while o_inst.communicatorInst is None:
         time.sleep(0.1)
 
-    # print nice header thing :)
-    if o_inst.onionrCore.config.get('general.display_header', True):
-        o_inst.header()
-
-    # print out debug info
-    o_inst.version(verbosity = 5, function = logger.debug)
-    logger.debug('Python version %s' % platform.python_version())
-
     logger.debug('Started communicator.')
 
     events.event('daemon_start', onionr = o_inst)
@@ -124,7 +118,7 @@ def kill_daemon(o_inst):
         Shutdown the Onionr daemon
     '''
 
-    logger.warn('Stopping the running daemon...', timestamp = False)
+    logger.warn('Stopping the running daemon...', timestamp = False, terminal=True)
     try:
         events.event('daemon_stop', onionr = o_inst)
         net = NetController(o_inst.onionrCore.config.get('client.port', 59496))
@@ -135,12 +129,12 @@ def kill_daemon(o_inst):
 
         net.killTor()
     except Exception as e:
-        logger.error('Failed to shutdown daemon.', error = e, timestamp = False)
+        logger.error('Failed to shutdown daemon.', error = e, timestamp = False, terminal=True)
     return
 
 def start(o_inst, input = False, override = False):
     if os.path.exists('.onionr-lock') and not override:
-        logger.fatal('Cannot start. Daemon is already running, or it did not exit cleanly.\n(if you are sure that there is not a daemon running, delete .onionr-lock & try again).')
+        logger.fatal('Cannot start. Daemon is already running, or it did not exit cleanly.\n(if you are sure that there is not a daemon running, delete .onionr-lock & try again).', terminal=True)
     else:
         if not o_inst.debug and not o_inst._developmentMode:
             lockFile = open('.onionr-lock', 'w')
diff --git a/onionr/onionrcommands/onionrstatistics.py b/onionr/onionrcommands/onionrstatistics.py
index 918d8445..a28fb7db 100755
--- a/onionr/onionrcommands/onionrstatistics.py
+++ b/onionr/onionrcommands/onionrstatistics.py
@@ -65,21 +65,21 @@ def show_stats(o_inst):
         groupsize = width - prewidth - len('[+] ')
 
         # generate stats table
-        logger.info(colors['title'] + 'Onionr v%s Statistics' % onionr.ONIONR_VERSION + colors['reset'])
-        logger.info(colors['border'] + '-' * (maxlength + 1) + '+' + colors['reset'])
+        logger.info(colors['title'] + 'Onionr v%s Statistics' % onionr.ONIONR_VERSION + colors['reset'], terminal=True)
+        logger.info(colors['border'] + '-' * (maxlength + 1) + '+' + colors['reset'], terminal=True)
         for key, val in messages.items():
             if not (type(val) is bool and val is True):
                 val = [str(val)[i:i + groupsize] for i in range(0, len(str(val)), groupsize)]
 
-                logger.info(colors['key'] + str(key).rjust(maxlength) + colors['reset'] + colors['border'] + ' | ' + colors['reset'] + colors['val'] + str(val.pop(0)) + colors['reset'])
+                logger.info(colors['key'] + str(key).rjust(maxlength) + colors['reset'] + colors['border'] + ' | ' + colors['reset'] + colors['val'] + str(val.pop(0)) + colors['reset'], terminal=True)
 
                 for value in val:
-                    logger.info(' ' * maxlength + colors['border'] + ' | ' + colors['reset'] + colors['val'] + str(value) + colors['reset'])
+                    logger.info(' ' * maxlength + colors['border'] + ' | ' + colors['reset'] + colors['val'] + str(value) + colors['reset'], terminal=True)
             else:
-                logger.info(colors['border'] + '-' * (maxlength + 1) + '+' + colors['reset'])
-        logger.info(colors['border'] + '-' * (maxlength + 1) + '+' + colors['reset'])
+                logger.info(colors['border'] + '-' * (maxlength + 1) + '+' + colors['reset'], terminal=True)
+        logger.info(colors['border'] + '-' * (maxlength + 1) + '+' + colors['reset'], terminal=True)
     except Exception as e:
-        logger.error('Failed to generate statistics table.', error = e, timestamp = False)
+        logger.error('Failed to generate statistics table.', error = e, timestamp = False, terminal=True)
 
 def show_details(o_inst):
     details = {
@@ -90,7 +90,7 @@ def show_details(o_inst):
     }
 
     for detail in details:
-        logger.info('%s%s: \n%s%s\n' % (logger.colors.fg.lightgreen, detail, logger.colors.fg.green, details[detail]), sensitive = True)
+        logger.info('%s%s: \n%s%s\n' % (logger.colors.fg.lightgreen, detail, logger.colors.fg.green, details[detail]), terminal = True)
 
 def show_peers(o_inst):
     randID = str(uuid.uuid4())
diff --git a/onionr/onionrcommands/openwebinterface.py b/onionr/onionrcommands/openwebinterface.py
index 2bd0cdd3..3e68e3ea 100755
--- a/onionr/onionrcommands/openwebinterface.py
+++ b/onionr/onionrcommands/openwebinterface.py
@@ -26,5 +26,5 @@ def open_home(o_inst):
         logger.error('Onionr seems to not be running (could not get api host)')
     else:
         url = 'http://%s/#%s' % (url, o_inst.onionrCore.config.get('client.webpassword'))
-        logger.info('If Onionr does not open automatically, use this URL: ' + url)
+        logger.info('If Onionr does not open automatically, use this URL: ' + url, terminal=True)
         webbrowser.open_new_tab(url)
\ No newline at end of file
diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py
index 887c421a..4aa02d76 100755
--- a/onionr/onionrutils.py
+++ b/onionr/onionrutils.py
@@ -519,7 +519,7 @@ class OnionrUtils:
         except KeyboardInterrupt:
             raise KeyboardInterrupt
         except ValueError as e:
-            logger.debug('Failed to make GET request to %s' % url, error = e, sensitive = True)
+            pass
         except onionrexceptions.InvalidAPIVersion:
             if 'X-API' in response_headers:
                 logger.debug('Using API version %s. Cannot communicate with node\'s API version of %s.' % (API_VERSION, response_headers['X-API']))
diff --git a/onionr/static-data/default-plugins/flow/main.py b/onionr/static-data/default-plugins/flow/main.py
index de406207..0162019f 100755
--- a/onionr/static-data/default-plugins/flow/main.py
+++ b/onionr/static-data/default-plugins/flow/main.py
@@ -40,7 +40,7 @@ class OnionrFlow:
         return
 
     def start(self):
-        logger.warn("Please note: everything said here is public, even if a random channel name is used.")
+        logger.warn("Please note: everything said here is public, even if a random channel name is used.", terminal=True)
         message = ""
         self.flowRunning = True
         newThread = threading.Thread(target=self.showOutput)
@@ -63,7 +63,7 @@ class OnionrFlow:
                 if len(message) > 0:
                     self.myCore.insertBlock(message, header='txt', expire=expireTime, meta={'ch': self.channel})
 
-        logger.info("Flow is exiting, goodbye")
+        logger.info("Flow is exiting, goodbye", terminal=True)
         return
 
     def showOutput(self):
@@ -81,11 +81,11 @@ class OnionrFlow:
                         continue
                     if not self.flowRunning:
                         break
-                    logger.info('\n------------------------', prompt = False)
+                    logger.info('\n------------------------', prompt = False, terminal=True)
                     content = block.getContent()
                     # Escape new lines, remove trailing whitespace, and escape ansi sequences
                     content = self.myCore._utils.escapeAnsi(content.replace('\n', '\\n').replace('\r', '\\r').strip())
-                    logger.info(block.getDate().strftime("%m/%d %H:%M") + ' - ' + logger.colors.reset + content, prompt = False)
+                    logger.info(block.getDate().strftime("%m/%d %H:%M") + ' - ' + logger.colors.reset + content, prompt = False, terminal=True)
                     self.alreadyOutputed.append(block.getHash())
                 time.sleep(5)
         except KeyboardInterrupt:

From 6630071802eb64eb385e8d27242522b612c4de6c Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Wed, 19 Jun 2019 19:59:05 -0500
Subject: [PATCH 03/29] log to terminal again in places where it should

---
 onionr/communicatorutils/lookupblocks.py      |   5 +-
 onionr/onionr.py                              |   4 +-
 onionr/onionrcommands/exportblocks.py         |   9 +-
 onionr/onionrcommands/filecommands.py         |  18 +--
 onionr/onionrcommands/keyadders.py            |  14 +--
 onionr/onionrcommands/openwebinterface.py     |   2 +-
 onionr/onionrcommands/plugincommands.py       |  22 ++--
 onionr/onionrcommands/pubkeymanager.py        |  36 +++---
 onionr/onionrcommands/resettor.py             |   2 +-
 .../static-data/default-plugins/cliui/main.py |   2 +-
 .../default-plugins/encrypt/main.py           |  22 ++--
 .../default-plugins/pluginmanager/main.py     | 108 +++++++++---------
 .../static-data/default-plugins/pms/main.py   |  40 +++----
 13 files changed, 144 insertions(+), 140 deletions(-)

diff --git a/onionr/communicatorutils/lookupblocks.py b/onionr/communicatorutils/lookupblocks.py
index e39e3c68..490a051e 100755
--- a/onionr/communicatorutils/lookupblocks.py
+++ b/onionr/communicatorutils/lookupblocks.py
@@ -78,6 +78,9 @@ def lookup_blocks_from_communicator(comm_inst):
                                 if len(comm_inst.blockQueue[i]) < 10:
                                     comm_inst.blockQueue[i].append(peer)
     if new_block_count > 0:
-        logger.info('Discovered %s new blocks' % (new_block_count,), terminal=True)
+        block_string = ""
+        if new_block_count > 1:
+            block_string = "s"
+        logger.info('Discovered %s new block%s' % (new_block_count, block_string), terminal=True)
     comm_inst.decrementThreadCount('lookupBlocks')
     return
\ No newline at end of file
diff --git a/onionr/onionr.py b/onionr/onionr.py
index 6306acca..e94e35b8 100755
--- a/onionr/onionr.py
+++ b/onionr/onionr.py
@@ -200,7 +200,7 @@ class Onionr:
     '''
 
     def exportBlock(self):
-        commands.exportblocks(self)
+        commands.exportblocks.export_block(self)
 
     def showDetails(self):
         commands.onionrstatistics.show_details(self)
@@ -347,7 +347,7 @@ class Onionr:
             Displays a "command not found" message
         '''
 
-        logger.error('Command not found.', timestamp = False)
+        logger.error('Command not found.', timestamp = False, terminal=True)
 
     def showHelpSuggestion(self):
         '''
diff --git a/onionr/onionrcommands/exportblocks.py b/onionr/onionrcommands/exportblocks.py
index 243ee752..daa70f35 100755
--- a/onionr/onionrcommands/exportblocks.py
+++ b/onionr/onionrcommands/exportblocks.py
@@ -17,7 +17,7 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import sys
+import sys, os
 import logger, onionrstorage
 def doExport(o_inst, bHash):
     exportDir = o_inst.dataDir + 'block-export/'
@@ -25,18 +25,19 @@ def doExport(o_inst, bHash):
         if os.path.exists(o_inst.dataDir):
             os.mkdir(exportDir)
         else:
-            logger.error('Onionr Not initialized')
+            logger.error('Onionr Not initialized', terminal=True)
     data = onionrstorage.getData(o_inst.onionrCore, bHash)
     with open('%s/%s.dat' % (exportDir, bHash), 'wb') as exportFile:
         exportFile.write(data)
+        logger.info('Block exported as file', terminal=True)
 
 def export_block(o_inst):
     exportDir = o_inst.dataDir + 'block-export/'
     try:
         assert o_inst.onionrUtils.validateHash(sys.argv[2])
     except (IndexError, AssertionError):
-        logger.error('No valid block hash specified.')
+        logger.error('No valid block hash specified.', terminal=True)
         sys.exit(1)
     else:
         bHash = sys.argv[2]
-        o_inst.doExport(bHash)
\ No newline at end of file
+        doExport(o_inst, bHash)
\ No newline at end of file
diff --git a/onionr/onionrcommands/filecommands.py b/onionr/onionrcommands/filecommands.py
index 444fd147..8f97f90a 100755
--- a/onionr/onionrcommands/filecommands.py
+++ b/onionr/onionrcommands/filecommands.py
@@ -31,18 +31,18 @@ def add_file(o_inst, singleBlock=False, blockType='bin'):
         contents = None
 
         if not os.path.exists(filename):
-            logger.error('That file does not exist. Improper path (specify full path)?')
+            logger.error('That file does not exist. Improper path (specify full path)?', terminal=True)
             return
-        logger.info('Adding file... this might take a long time.')
+        logger.info('Adding file... this might take a long time.', terminal=True)
         try:
             with open(filename, 'rb') as singleFile:
                 blockhash = o_inst.onionrCore.insertBlock(base64.b64encode(singleFile.read()), header=blockType)
             if len(blockhash) > 0:
-                logger.info('File %s saved in block %s' % (filename, blockhash))
+                logger.info('File %s saved in block %s' % (filename, blockhash), terminal=True)
         except:
-            logger.error('Failed to save file in block.', timestamp = False)
+            logger.error('Failed to save file in block.', timestamp = False, terminal=True)
     else:
-        logger.error('%s add-file <filename>' % sys.argv[0], timestamp = False)
+        logger.error('%s add-file <filename>' % sys.argv[0], timestamp = False, terminal=True)
 
 def getFile(o_inst):
     '''
@@ -52,16 +52,16 @@ def getFile(o_inst):
         fileName = sys.argv[2]
         bHash = sys.argv[3]
     except IndexError:
-        logger.error("Syntax %s %s" % (sys.argv[0], '/path/to/filename <blockhash>'))
+        logger.error("Syntax %s %s" % (sys.argv[0], '/path/to/filename <blockhash>'), terminal=True)
     else:
-        logger.info(fileName)
+        logger.info(fileName, terminal=True)
 
         contents = None
         if os.path.exists(fileName):
-            logger.error("File already exists")
+            logger.error("File already exists", terminal=True)
             return
         if not o_inst.onionrUtils.validateHash(bHash):
-            logger.error('Block hash is invalid')
+            logger.error('Block hash is invalid', terminal=True)
             return
 
         with open(fileName, 'wb') as myFile:
diff --git a/onionr/onionrcommands/keyadders.py b/onionr/onionrcommands/keyadders.py
index 0edda6b0..363dfe28 100755
--- a/onionr/onionrcommands/keyadders.py
+++ b/onionr/onionrcommands/keyadders.py
@@ -26,14 +26,14 @@ def add_peer(o_inst):
         pass
     else:
         if o_inst.onionrUtils.hasKey(newPeer):
-            logger.info('We already have that key')
+            logger.info('We already have that key', terminal=True)
             return
-        logger.info("Adding peer: " + logger.colors.underline + newPeer)
+        logger.info("Adding peer: " + logger.colors.underline + newPeer, terminal=True)
         try:
             if o_inst.onionrCore.addPeer(newPeer):
-                logger.info('Successfully added key')
+                logger.info('Successfully added key', terminal=True)
         except AssertionError:
-            logger.error('Failed to add key')
+            logger.error('Failed to add key', terminal=True)
 
 def add_address(o_inst):
     try:
@@ -42,8 +42,8 @@ def add_address(o_inst):
     except IndexError:
         pass
     else:
-        logger.info("Adding address: " + logger.colors.underline + newAddress)
+        logger.info("Adding address: " + logger.colors.underline + newAddress, terminal=True)
         if o_inst.onionrCore.addAddress(newAddress):
-            logger.info("Successfully added address.")
+            logger.info("Successfully added address.", terminal=True)
         else:
-            logger.warn("Unable to add address.")
\ No newline at end of file
+            logger.warn("Unable to add address.", terminal=True)
\ No newline at end of file
diff --git a/onionr/onionrcommands/openwebinterface.py b/onionr/onionrcommands/openwebinterface.py
index 3e68e3ea..8ce71744 100755
--- a/onionr/onionrcommands/openwebinterface.py
+++ b/onionr/onionrcommands/openwebinterface.py
@@ -23,7 +23,7 @@ def open_home(o_inst):
     try:
         url = o_inst.onionrUtils.getClientAPIServer()
     except FileNotFoundError:
-        logger.error('Onionr seems to not be running (could not get api host)')
+        logger.error('Onionr seems to not be running (could not get api host)', terminal=True)
     else:
         url = 'http://%s/#%s' % (url, o_inst.onionrCore.config.get('client.webpassword'))
         logger.info('If Onionr does not open automatically, use this URL: ' + url, terminal=True)
diff --git a/onionr/onionrcommands/plugincommands.py b/onionr/onionrcommands/plugincommands.py
index e4f88d96..154bb6cf 100755
--- a/onionr/onionrcommands/plugincommands.py
+++ b/onionr/onionrcommands/plugincommands.py
@@ -24,18 +24,18 @@ import logger, onionrplugins as plugins
 def enable_plugin(o_inst):
     if len(sys.argv) >= 3:
         plugin_name = sys.argv[2]
-        logger.info('Enabling plugin "%s"...' % plugin_name)
+        logger.info('Enabling plugin "%s"...' % plugin_name, terminal=True)
         plugins.enable(plugin_name, o_inst)
     else:
-        logger.info('%s %s <plugin>' % (sys.argv[0], sys.argv[1]))
+        logger.info('%s %s <plugin>' % (sys.argv[0], sys.argv[1]), terminal=True)
 
 def disable_plugin(o_inst):
     if len(sys.argv) >= 3:
         plugin_name = sys.argv[2]
-        logger.info('Disabling plugin "%s"...' % plugin_name)
+        logger.info('Disabling plugin "%s"...' % plugin_name, terminal=True)
         plugins.disable(plugin_name, o_inst)
     else:
-        logger.info('%s %s <plugin>' % (sys.argv[0], sys.argv[1]))
+        logger.info('%s %s <plugin>' % (sys.argv[0], sys.argv[1]), terminal=True)
 
 def reload_plugin(o_inst):
     '''
@@ -44,11 +44,11 @@ def reload_plugin(o_inst):
 
     if len(sys.argv) >= 3:
         plugin_name = sys.argv[2]
-        logger.info('Reloading plugin "%s"...' % plugin_name)
+        logger.info('Reloading plugin "%s"...' % plugin_name, terminal=True)
         plugins.stop(plugin_name, o_inst)
         plugins.start(plugin_name, o_inst)
     else:
-        logger.info('Reloading all plugins...')
+        logger.info('Reloading all plugins...', terminal=True)
         plugins.reload(o_inst)
 
 
@@ -62,7 +62,7 @@ def create_plugin(o_inst):
             plugin_name = re.sub('[^0-9a-zA-Z_]+', '', str(sys.argv[2]).lower())
 
             if not plugins.exists(plugin_name):
-                logger.info('Creating plugin "%s"...' % plugin_name)
+                logger.info('Creating plugin "%s"...' % plugin_name, terminal=True)
 
                 os.makedirs(plugins.get_plugins_folder(plugin_name))
                 with open(plugins.get_plugins_folder(plugin_name) + '/main.py', 'a') as main:
@@ -76,12 +76,12 @@ def create_plugin(o_inst):
                 with open(plugins.get_plugins_folder(plugin_name) + '/info.json', 'a') as main:
                     main.write(json.dumps({'author' : 'anonymous', 'description' : 'the default description of the plugin', 'version' : '1.0'}))
 
-                logger.info('Enabling plugin "%s"...' % plugin_name)
+                logger.info('Enabling plugin "%s"...' % plugin_name, terminal=True)
                 plugins.enable(plugin_name, o_inst)
             else:
-                logger.warn('Cannot create plugin directory structure; plugin "%s" exists.' % plugin_name)
+                logger.warn('Cannot create plugin directory structure; plugin "%s" exists.' % plugin_name, terminal=True)
 
         except Exception as e:
-            logger.error('Failed to create plugin directory structure.', e)
+            logger.error('Failed to create plugin directory structure.', e, terminal=True)
     else:
-        logger.info('%s %s <plugin>' % (sys.argv[0], sys.argv[1]))
\ No newline at end of file
+        logger.info('%s %s <plugin>' % (sys.argv[0], sys.argv[1]), terminal=True)
\ No newline at end of file
diff --git a/onionr/onionrcommands/pubkeymanager.py b/onionr/onionrcommands/pubkeymanager.py
index 2a6370f9..e8f29b2a 100755
--- a/onionr/onionrcommands/pubkeymanager.py
+++ b/onionr/onionrcommands/pubkeymanager.py
@@ -29,42 +29,42 @@ def add_ID(o_inst):
     except (IndexError, AssertionError) as e:
         newID = o_inst.onionrCore._crypto.keyManager.addKey()[0]
     else:
-        logger.warn('Deterministic keys require random and long passphrases.')
-        logger.warn('If a good passphrase is not used, your key can be easily stolen.')
-        logger.warn('You should use a series of hard to guess words, see this for reference: https://www.xkcd.com/936/')
+        logger.warn('Deterministic keys require random and long passphrases.', terminal=True)
+        logger.warn('If a good passphrase is not used, your key can be easily stolen.', terminal=True)
+        logger.warn('You should use a series of hard to guess words, see this for reference: https://www.xkcd.com/936/', terminal=True)
         pass1 = getpass.getpass(prompt='Enter at least %s characters: ' % (o_inst.onionrCore._crypto.deterministicRequirement,))
         pass2 = getpass.getpass(prompt='Confirm entry: ')
         if o_inst.onionrCore._crypto.safeCompare(pass1, pass2):
             try:
-                logger.info('Generating deterministic key. This can take a while.')
+                logger.info('Generating deterministic key. This can take a while.', terminal=True)
                 newID, privKey = o_inst.onionrCore._crypto.generateDeterministic(pass1)
             except onionrexceptions.PasswordStrengthError:
-                logger.error('Passphrase must use at least %s characters.' % (o_inst.onionrCore._crypto.deterministicRequirement,))
+                logger.error('Passphrase must use at least %s characters.' % (o_inst.onionrCore._crypto.deterministicRequirement,), terminal=True)
                 sys.exit(1)
         else:
-            logger.error('Passwords do not match.')
+            logger.error('Passwords do not match.', terminal=True)
             sys.exit(1)
         o_inst.onionrCore._crypto.keyManager.addKey(pubKey=newID, 
         privKey=privKey)
-    logger.info('Added ID: %s' % (o_inst.onionrUtils.bytesToStr(newID),))
+    logger.info('Added ID: %s' % (o_inst.onionrUtils.bytesToStr(newID),), terminal=True)
 
 def change_ID(o_inst):
     try:
         key = sys.argv[2]
         key = unpaddedbase32.repad(key.encode()).decode()
     except IndexError:
-        logger.warn('Specify pubkey to use')
+        logger.warn('Specify pubkey to use', terminal=True)
     else:
         if o_inst.onionrUtils.validatePubKey(key):
             if key in o_inst.onionrCore._crypto.keyManager.getPubkeyList():
                 o_inst.onionrCore.config.set('general.public_key', key)
                 o_inst.onionrCore.config.save()
-                logger.info('Set active key to: %s' % (key,))
-                logger.info('Restart Onionr if it is running.')
+                logger.info('Set active key to: %s' % (key,), terminal=True)
+                logger.info('Restart Onionr if it is running.', terminal=True)
             else:
-                logger.warn('That key does not exist')
+                logger.warn('That key does not exist', terminal=True)
         else:
-            logger.warn('Invalid key %s' % (key,))
+            logger.warn('Invalid key %s' % (key,), terminal=True)
 
 def friend_command(o_inst):
     friend = ''
@@ -72,13 +72,13 @@ def friend_command(o_inst):
         # Get the friend command
         action = sys.argv[2]
     except IndexError:
-        logger.info('Syntax: friend add/remove/list [address]')
+        logger.info('Syntax: friend add/remove/list [address]', terminal=True)
     else:
         action = action.lower()
         if action == 'list':
             # List out peers marked as our friend
             for friend in contactmanager.ContactManager.list_friends(o_inst.onionrCore):
-                logger.info(friend.publicKey + ' - ' + friend.get_info('name'))
+                logger.info(friend.publicKey + ' - ' + friend.get_info('name'), terminal=True)
         elif action in ('add', 'remove'):
             try:
                 friend = sys.argv[3]
@@ -88,7 +88,7 @@ def friend_command(o_inst):
                     raise onionrexceptions.KeyNotKnown
                 friend = onionrusers.OnionrUser(o_inst.onionrCore, friend)
             except IndexError:
-                logger.warn('Friend ID is required.')
+                logger.warn('Friend ID is required.', terminal=True)
                 action = 'error' # set to 'error' so that the finally block does not process anything
             except onionrexceptions.KeyNotKnown:
                 o_inst.onionrCore.addPeer(friend)
@@ -96,9 +96,9 @@ def friend_command(o_inst):
             finally:
                 if action == 'add':
                     friend.setTrust(1)
-                    logger.info('Added %s as friend.' % (friend.publicKey,))
+                    logger.info('Added %s as friend.' % (friend.publicKey,), terminal=True)
                 elif action == 'remove':
                     friend.setTrust(0)
-                    logger.info('Removed %s as friend.' % (friend.publicKey,))
+                    logger.info('Removed %s as friend.' % (friend.publicKey,), terminal=True)
         else:
-            logger.info('Syntax: friend add/remove/list [address]')
\ No newline at end of file
+            logger.info('Syntax: friend add/remove/list [address]', terminal=True)
\ No newline at end of file
diff --git a/onionr/onionrcommands/resettor.py b/onionr/onionrcommands/resettor.py
index d49cacfe..cd7c51d2 100755
--- a/onionr/onionrcommands/resettor.py
+++ b/onionr/onionrcommands/resettor.py
@@ -24,6 +24,6 @@ def reset_tor():
     tor_dir = c.dataDir + 'tordata'
     if os.path.exists(tor_dir):
         if c._utils.localCommand('/ping') == 'pong!':
-            logger.warn('Cannot delete Tor data while Onionr is running')
+            logger.warn('Cannot delete Tor data while Onionr is running', terminal=True)
         else:
             shutil.rmtree(tor_dir)
\ No newline at end of file
diff --git a/onionr/static-data/default-plugins/cliui/main.py b/onionr/static-data/default-plugins/cliui/main.py
index c4186b81..5f47c455 100755
--- a/onionr/static-data/default-plugins/cliui/main.py
+++ b/onionr/static-data/default-plugins/cliui/main.py
@@ -100,7 +100,7 @@ class OnionrCLIUI:
             elif choice == "":
                 pass
             else:
-                logger.error("Invalid choice")
+                logger.error("Invalid choice", terminal=True)
         return
 
 def on_init(api, data = None):
diff --git a/onionr/static-data/default-plugins/encrypt/main.py b/onionr/static-data/default-plugins/encrypt/main.py
index 15697ac2..8c874c99 100755
--- a/onionr/static-data/default-plugins/encrypt/main.py
+++ b/onionr/static-data/default-plugins/encrypt/main.py
@@ -1,5 +1,5 @@
 '''
-    Onionr - P2P Microblogging Platform & Social network
+    Onionr - Private P2P Communication
 
     This default plugin allows users to encrypt/decrypt messages without using blocks
 '''
@@ -46,13 +46,13 @@ class PlainEncryption:
             if not self.api.get_core()._utils.validatePubKey(sys.argv[2]):
                 raise onionrexceptions.InvalidPubkey
         except (ValueError, IndexError) as e:
-            logger.error("Peer public key not specified")
+            logger.error("Peer public key not specified", terminal=True)
         except onionrexceptions.InvalidPubkey:
-            logger.error("Invalid public key")
+            logger.error("Invalid public key", terminal=True)
         else:
             pubkey = sys.argv[2]
             # Encrypt if public key is valid
-            logger.info("Please enter your message (ctrl-d or -q to stop):")
+            logger.info("Please enter your message (ctrl-d or -q to stop):", terminal=True)
             try:
                 for line in sys.stdin:
                     if line == '-q\n':
@@ -72,12 +72,12 @@ class PlainEncryption:
             plaintext = data
             encrypted = self.api.get_core()._crypto.pubKeyEncrypt(plaintext, pubkey, encodedData=True)
             encrypted = self.api.get_core()._utils.bytesToStr(encrypted)
-            logger.info('Encrypted Message: \n\nONIONR ENCRYPTED DATA %s END ENCRYPTED DATA' % (encrypted,))
+            logger.info('Encrypted Message: \n\nONIONR ENCRYPTED DATA %s END ENCRYPTED DATA' % (encrypted,), terminal=True)
 
     def decrypt(self):
         plaintext = ""
         data = ""
-        logger.info("Please enter your message (ctrl-d or -q to stop):")
+        logger.info("Please enter your message (ctrl-d or -q to stop):", terminal=True)
         try:
             for line in sys.stdin:
                 if line == '-q\n':
@@ -91,17 +91,17 @@ class PlainEncryption:
         myPub = self.api.get_core()._crypto.pubKey
         decrypted = self.api.get_core()._crypto.pubKeyDecrypt(encrypted, privkey=self.api.get_core()._crypto.privKey, encodedData=True)
         if decrypted == False:
-            logger.error("Decryption failed")
+            logger.error("Decryption failed", terminal=True)
         else:
             data = json.loads(decrypted)
-            logger.info('Decrypted Message: \n\n%s' % data['data'])
+            logger.info('Decrypted Message: \n\n%s' % data['data'], terminal=True)
             try:
-                logger.info("Signing public key: %s" % (data['signer'],))
+                logger.info("Signing public key: %s" % (data['signer'],), terminal=True)
                 assert self.api.get_core()._crypto.edVerify(data['data'], data['signer'], data['sig']) != False
             except (AssertionError, KeyError) as e:
-                logger.warn("WARNING: THIS MESSAGE HAS A MISSING OR INVALID SIGNATURE")
+                logger.warn("WARNING: THIS MESSAGE HAS A MISSING OR INVALID SIGNATURE", terminal=True)
             else:
-                logger.info("Message has good signature.")
+                logger.info("Message has good signature.", terminal=True)
         return
 
 def on_init(api, data = None):
diff --git a/onionr/static-data/default-plugins/pluginmanager/main.py b/onionr/static-data/default-plugins/pluginmanager/main.py
index 979b1133..d4feb69d 100755
--- a/onionr/static-data/default-plugins/pluginmanager/main.py
+++ b/onionr/static-data/default-plugins/pluginmanager/main.py
@@ -1,5 +1,5 @@
 '''
-    Onionr - P2P Microblogging Platform & Social network.
+    Onionr - Private P2P Communication
 
     This plugin acts as a plugin manager, and allows the user to install other plugins distributed over Onionr.
 '''
@@ -180,11 +180,11 @@ def blockToPlugin(block):
         shutil.unpack_archive(source, destination)
         pluginapi.plugins.enable(name)
 
-        logger.info('Installation of %s complete.' % name)
+        logger.info('Installation of %s complete.' % name, terminal=True)
 
         return True
     except Exception as e:
-        logger.error('Failed to install plugin.', error = e, timestamp = False)
+        logger.error('Failed to install plugin.', error = e, timestamp = False, terminal=True)
 
     return False
 
@@ -240,9 +240,9 @@ def pluginToBlock(plugin, import_block = True):
 
             return hash
         else:
-            logger.error('Plugin %s does not exist.' % plugin)
+            logger.error('Plugin %s does not exist.' % plugin, terminal=True)
     except Exception as e:
-        logger.error('Failed to convert plugin to block.', error = e, timestamp = False)
+        logger.error('Failed to convert plugin to block.', error = e, timestamp = False, terminal=True)
 
     return False
 
@@ -261,7 +261,7 @@ def installBlock(block):
 
         install = False
 
-        logger.info(('Will install %s' + (' v' + version if not version is None else '') + ' (%s), by %s') % (name, date, author))
+        logger.info(('Will install %s' + (' v' + version if not version is None else '') + ' (%s), by %s') % (name, date, author), terminal=True)
 
         # TODO: Convert to single line if statement
         if os.path.exists(pluginapi.plugins.get_folder(name)):
@@ -273,12 +273,12 @@ def installBlock(block):
             blockToPlugin(block.getHash())
             addPlugin(name)
         else:
-            logger.info('Installation cancelled.')
+            logger.info('Installation cancelled.', terminal=True)
             return False
 
         return True
     except Exception as e:
-        logger.error('Failed to install plugin.', error = e, timestamp = False)
+        logger.error('Failed to install plugin.', error = e, timestamp = False, terminal=True)
         return False
 
 def uninstallPlugin(plugin):
@@ -291,12 +291,12 @@ def uninstallPlugin(plugin):
         remove = False
 
         if not exists:
-            logger.warn('Plugin %s does not exist.' % plugin, timestamp = False)
+            logger.warn('Plugin %s does not exist.' % plugin, timestamp = False, terminal=True)
             return False
 
         default = 'y'
         if not installedByPluginManager:
-            logger.warn('The plugin %s was not installed by %s.' % (plugin, plugin_name), timestamp = False)
+            logger.warn('The plugin %s was not installed by %s.' % (plugin, plugin_name), timestamp = False, terminal=True)
             default = 'n'
         remove = logger.confirm(message = 'All plugin data will be lost. Are you sure you want to proceed %s?', default = default)
 
@@ -306,20 +306,20 @@ def uninstallPlugin(plugin):
             pluginapi.plugins.disable(plugin)
             shutil.rmtree(pluginFolder)
 
-            logger.info('Uninstallation of %s complete.' % plugin)
+            logger.info('Uninstallation of %s complete.' % plugin, terminal=True)
 
             return True
         else:
             logger.info('Uninstallation cancelled.')
     except Exception as e:
-        logger.error('Failed to uninstall plugin.', error = e)
+        logger.error('Failed to uninstall plugin.', error = e, terminal=True)
     return False
 
 # command handlers
 
 def help():
-    logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin> [public key/block hash]')
-    logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin> [public key/block hash]')
+    logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin> [public key/block hash]', terminal=True)
+    logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin> [public key/block hash]', terminal=True)
 
 def commandInstallPlugin():
     if len(sys.argv) >= 3:
@@ -345,20 +345,20 @@ def commandInstallPlugin():
 
         if pkobh is None:
             # still nothing found, try searching repositories
-            logger.info('Searching for public key in repositories...')
+            logger.info('Searching for public key in repositories...', terminal=True)
             try:
                 repos = getRepositories()
                 distributors = list()
                 for repo, records in repos.items():
                     if pluginname in records:
-                        logger.debug('Found %s in repository %s for plugin %s.' % (records[pluginname], repo, pluginname))
+                        logger.debug('Found %s in repository %s for plugin %s.' % (records[pluginname], repo, pluginname), terminal=True)
                         distributors.append(records[pluginname])
 
                 if len(distributors) != 0:
                     distributor = None
 
                     if len(distributors) == 1:
-                        logger.info('Found distributor: %s' % distributors[0])
+                        logger.info('Found distributor: %s' % distributors[0], terminal=True)
                         distributor = distributors[0]
                     else:
                         distributors_message = ''
@@ -368,11 +368,11 @@ def commandInstallPlugin():
                             distributors_message += '    ' + logger.colors.bold + str(index) + ') ' + logger.colors.reset + str(dist) + '\n'
                             index += 1
 
-                        logger.info((logger.colors.bold + 'Found distributors (%s):' + logger.colors.reset + '\n' + distributors_message) % len(distributors))
+                        logger.info((logger.colors.bold + 'Found distributors (%s):' + logger.colors.reset + '\n' + distributors_message) % len(distributors), terminal=True)
 
                         valid = False
                         while not valid:
-                            choice = logger.readline('Select the number of the key to use, from 1 to %s, or press Ctrl+C to cancel:' % (index - 1))
+                            choice = logger.readline('Select the number of the key to use, from 1 to %s, or press Ctrl+C to cancel:' % (index - 1), terminal=True)
 
                             try:
                                 choice = int(choice)
@@ -380,7 +380,7 @@ def commandInstallPlugin():
                                     distributor = distributors[int(choice)]
                                     valid = True
                             except KeyboardInterrupt:
-                                logger.info('Installation cancelled.')
+                                logger.info('Installation cancelled.', terminal=True)
                                 return True
                             except:
                                 pass
@@ -388,11 +388,11 @@ def commandInstallPlugin():
                     if not distributor is None:
                         pkobh = distributor
             except Exception as e:
-                logger.warn('Failed to lookup plugin in repositories.', timestamp = False)
+                logger.warn('Failed to lookup plugin in repositories.', timestamp = False, terminal=True)
                 return True
 
         if pkobh is None:
-            logger.error('No key for this plugin found in keystore or repositories, please specify.', timestamp = False)
+            logger.error('No key for this plugin found in keystore or repositories, please specify.', timestamp = False, terminal=True)
 
             return True
 
@@ -409,21 +409,21 @@ def commandInstallPlugin():
         blockhash = None
 
         if valid_hash and not real_block:
-            logger.error('Block hash not found. Perhaps it has not been synced yet?', timestamp = False)
-            logger.debug('Is valid hash, but does not belong to a known block.')
+            logger.error('Block hash not found. Perhaps it has not been synced yet?', timestamp = False, terminal=True)
+            logger.debug('Is valid hash, but does not belong to a known block.', terminal=True)
 
             return True
         elif valid_hash and real_block:
             blockhash = str(pkobh)
-            logger.debug('Using block %s...' % blockhash)
+            logger.debug('Using block %s...' % blockhash, terminal=True)
 
             installBlock(blockhash)
         elif valid_key and not real_key:
-            logger.error('Public key not found. Try adding the node by address manually, if possible.', timestamp = False)
-            logger.debug('Is valid key, but the key is not a known one.')
+            logger.error('Public key not found. Try adding the node by address manually, if possible.', timestamp = False, terminal=True)
+            logger.debug('Is valid key, but the key is not a known one.', terminal=True)
         elif valid_key and real_key:
             publickey = str(pkobh)
-            logger.debug('Using public key %s...' % publickey)
+            logger.debug('Using public key %s...' % publickey, terminal=True)
 
             saveKey(pluginname, pkobh)
 
@@ -455,14 +455,14 @@ def commandInstallPlugin():
                 except Exception as e:
                     pass
 
-            logger.warn('Only continue the installation if you are absolutely certain that you trust the plugin distributor. Public key of plugin distributor: %s' % publickey, timestamp = False)
-            logger.debug('Most recent block matching parameters is %s' % mostRecentVersionBlock)
+            logger.warn('Only continue the installation if you are absolutely certain that you trust the plugin distributor. Public key of plugin distributor: %s' % publickey, timestamp = False, terminal=True)
+            logger.debug('Most recent block matching parameters is %s' % mostRecentVersionBlock, terminal=True)
             installBlock(mostRecentVersionBlock)
         else:
-            logger.error('Unknown data "%s"; must be public key or block hash.' % str(pkobh), timestamp = False)
+            logger.error('Unknown data "%s"; must be public key or block hash.' % str(pkobh), timestamp = False, terminal=True)
             return
     else:
-        logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin> [public key/block hash]')
+        logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin> [public key/block hash]', terminal=True)
 
     return True
 
@@ -470,12 +470,12 @@ def commandUninstallPlugin():
     if len(sys.argv) >= 3:
         uninstallPlugin(sys.argv[2])
     else:
-        logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin>')
+        logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin>', terminal=True)
 
     return True
 
 def commandSearchPlugin():
-    logger.info('This feature has not been created yet. Please check back later.')
+    logger.info('This feature has not been created yet. Please check back later.', terminal=True)
     return True
 
 def commandAddRepository():
@@ -495,22 +495,22 @@ def commandAddRepository():
                         if pluginapi.get_utils().validatePubKey(distributor):
                             pluginslist[pluginname] = distributor
 
-                    logger.debug('Found %s records in repository.' % len(pluginslist))
+                    logger.debug('Found %s records in repository.' % len(pluginslist), terminal=True)
 
                     if len(pluginslist) != 0:
                         addRepository(blockhash, pluginslist)
-                        logger.info('Successfully added repository.')
+                        logger.info('Successfully added repository.', terminal=True)
                     else:
-                        logger.error('Repository contains no records, not importing.', timestamp = False)
+                        logger.error('Repository contains no records, not importing.', timestamp = False, terminal=True)
                 except Exception as e:
-                    logger.error('Failed to parse block.', error = e)
+                    logger.error('Failed to parse block.', error = e, terminal=True)
             else:
-                logger.error('Block hash not found. Perhaps it has not been synced yet?', timestamp = False)
+                logger.error('Block hash not found. Perhaps it has not been synced yet?', timestamp = False, terminal=True)
                 logger.debug('Is valid hash, but does not belong to a known block.')
         else:
-            logger.error('Unknown data "%s"; must be block hash.' % str(pkobh), timestamp = False)
+            logger.error('Unknown data "%s"; must be block hash.' % str(pkobh), timestamp = False, terminal=True)
     else:
-        logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' [block hash]')
+        logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' [block hash]', terminal=True)
 
     return True
 
@@ -524,15 +524,15 @@ def commandRemoveRepository():
             if blockhash in getRepositories():
                 try:
                     removeRepository(blockhash)
-                    logger.info('Successfully removed repository.')
+                    logger.info('Successfully removed repository.', terminal=True)
                 except Exception as e:
-                    logger.error('Failed to parse block.', error = e)
+                    logger.error('Failed to parse block.', error = e, terminal=True)
             else:
-                logger.error('Repository has not been imported, nothing to remove.', timestamp = False)
+                logger.error('Repository has not been imported, nothing to remove.', timestamp = False, terminal=True)
         else:
-            logger.error('Unknown data "%s"; must be block hash.' % str(pkobh))
+            logger.error('Unknown data "%s"; must be block hash.' % str(pkobh), terminal=True)
     else:
-        logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' [block hash]')
+        logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' [block hash]', terminal=True)
 
     return True
 
@@ -545,11 +545,11 @@ def commandPublishPlugin():
 
         if os.path.exists(pluginfolder) and not os.path.isfile(pluginfolder):
             block = pluginToBlock(pluginname)
-            logger.info('Plugin saved in block %s.' % block)
+            logger.info('Plugin saved in block %s.' % block, terminal=True)
         else:
-            logger.error('Plugin %s does not exist.' % pluginname, timestamp = False)
+            logger.error('Plugin %s does not exist.' % pluginname, timestamp = False, terminal=True)
     else:
-        logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin>')
+        logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin>', terminal=True)
 
 def commandCreateRepository():
     if len(sys.argv) >= 3:
@@ -573,22 +573,22 @@ def commandCreateRepository():
             if distributor is None:
                 distributor = getKey(pluginname)
             if distributor is None:
-                logger.error('No distributor key was found for the plugin %s.' % pluginname, timestamp = False)
+                logger.error('No distributor key was found for the plugin %s.' % pluginname, timestamp = False, terminal=True)
                 success = False
 
             plugins.append([pluginname, distributor])
 
         if not success:
-            logger.error('Please correct the above errors, then recreate the repository.')
+            logger.error('Please correct the above errors, then recreate the repository.', terminal=True)
             return True
 
         blockhash = createRepository(plugins)
         if not blockhash is None:
-            logger.info('Successfully created repository. Execute the following command to add the repository:\n    ' + logger.colors.underline + '%s --add-repository %s' % (script, blockhash))
+            logger.info('Successfully created repository. Execute the following command to add the repository:\n    ' + logger.colors.underline + '%s --add-repository %s' % (script, blockhash), terminal=True)
         else:
-            logger.error('Failed to create repository, an unknown error occurred.')
+            logger.error('Failed to create repository, an unknown error occurred.', terminal=True)
     else:
-        logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' [plugins...]')
+        logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' [plugins...]', terminal=True)
 
     return True
 
diff --git a/onionr/static-data/default-plugins/pms/main.py b/onionr/static-data/default-plugins/pms/main.py
index 9acb99f2..e91a153c 100755
--- a/onionr/static-data/default-plugins/pms/main.py
+++ b/onionr/static-data/default-plugins/pms/main.py
@@ -73,7 +73,7 @@ class OnionrMail:
         blockCount = 0
         pmBlockMap = {}
         pmBlocks = {}
-        logger.info('Decrypting messages...')
+        logger.info('Decrypting messages...', terminal=True)
         choice = ''
         displayList = []
         subject = ''
@@ -108,7 +108,7 @@ class OnionrMail:
             displayList.append('%s. %s - %s - <%s>: %s' % (blockCount, blockDate, senderDisplay[:12], subject[:10], blockHash))
         while choice not in ('-q', 'q', 'quit'):
             for i in displayList:
-                logger.info(i)
+                logger.info(i, terminal=True)
             try:
                 choice = logger.readline('Enter a block number, -r to refresh, or -q to stop: ').strip().lower()
             except (EOFError, KeyboardInterrupt):
@@ -138,18 +138,18 @@ class OnionrMail:
                     senderDisplay = self.myCore._utils.bytesToStr(readBlock.signer)
                     if len(senderDisplay.strip()) == 0:
                         senderDisplay = 'Anonymous'
-                    logger.info('Message received from %s' % (senderDisplay,))
-                    logger.info('Valid signature: %s' % readBlock.validSig)
+                    logger.info('Message received from %s' % (senderDisplay,), terminal=True)
+                    logger.info('Valid signature: %s' % readBlock.validSig, terminal=True)
 
                     if not readBlock.validSig:
-                        logger.warn('This message has an INVALID/NO signature. ANYONE could have sent this message.')
+                        logger.warn('This message has an INVALID/NO signature. ANYONE could have sent this message.', terminal=True)
                         cancel = logger.readline('Press enter to continue to message, or -q to not open the message (recommended).')
                         print('')
                     if cancel != '-q':
                         try:
                             print(draw_border(self.myCore._utils.escapeAnsi(readBlock.bcontent.decode().strip())))
                         except ValueError:
-                            logger.warn('Error presenting message. This is usually due to a malformed or blank message.')
+                            logger.warn('Error presenting message. This is usually due to a malformed or blank message.', terminal=True)
                             pass
                         if readBlock.validSig:
                             reply = logger.readline("Press enter to continue, or enter %s to reply" % ("-r",))
@@ -168,7 +168,7 @@ class OnionrMail:
         entering = True
         while entering:
             self.get_sent_list()
-            logger.info('Enter a block number or -q to return')
+            logger.info('Enter a block number or -q to return', terminal=True)
             try:
                 choice = input('>')
             except (EOFError, KeyboardInterrupt) as e:
@@ -182,11 +182,11 @@ class OnionrMail:
                     try:
                         self.sentboxList[int(choice)]
                     except (IndexError, ValueError) as e:
-                        logger.warn('Invalid block.')
+                        logger.warn('Invalid block.', terminal=True)
                     else:
-                        logger.info('Sent to: ' + self.sentMessages[self.sentboxList[int(choice)]][1])
+                        logger.info('Sent to: ' + self.sentMessages[self.sentboxList[int(choice)]][1], terminal=True)
                         # Print ansi escaped sent message
-                        logger.info(self.myCore._utils.escapeAnsi(self.sentMessages[self.sentboxList[int(choice)]][0]))
+                        logger.info(self.myCore._utils.escapeAnsi(self.sentMessages[self.sentboxList[int(choice)]][0]), terminal=True)
                         input('Press enter to continue...')
                 finally:
                     if choice == '-q':
@@ -201,7 +201,7 @@ class OnionrMail:
             self.sentboxList.append(i['hash'])
             self.sentMessages[i['hash']] = (self.myCore._utils.bytesToStr(i['message']), i['peer'], i['subject'])
             if display:
-                logger.info('%s. %s - %s - (%s) - %s' % (count, i['hash'], i['peer'][:12], i['subject'], i['date']))
+                logger.info('%s. %s - %s - (%s) - %s' % (count, i['hash'], i['peer'][:12], i['subject'], i['date']), terminal=True)
             count += 1
         return json.dumps(self.sentMessages)
 
@@ -220,7 +220,7 @@ class OnionrMail:
                     if not self.myCore._utils.validatePubKey(recip):
                         raise onionrexceptions.InvalidPubkey('Must be a valid ed25519 base32 encoded public key')
                 except onionrexceptions.InvalidPubkey:
-                    logger.warn('Invalid public key')
+                    logger.warn('Invalid public key', terminal=True)
                 except (KeyboardInterrupt, EOFError):
                     entering = False
                 else:
@@ -234,7 +234,7 @@ class OnionrMail:
             pass
         
         cancelEnter = False
-        logger.info('Enter your message, stop by entering -q on a new line. -c to cancel')
+        logger.info('Enter your message, stop by entering -q on a new line. -c to cancel', terminal=True)
         while newLine != '-q':
             try:
                 newLine = input()
@@ -249,7 +249,7 @@ class OnionrMail:
             message += newLine
 
         if not cancelEnter:
-            logger.info('Inserting encrypted message as Onionr block....')
+            logger.info('Inserting encrypted message as Onionr block....', terminal=True)
 
             blockID = self.myCore.insertBlock(message, header='pm', encryptType='asym', asymPeer=recip, sign=self.doSigs, meta={'subject': subject})
     
@@ -261,16 +261,16 @@ class OnionrMail:
         while True:
             sigMsg = 'Message Signing: %s'
 
-            logger.info(self.strings.programTag + '\n\nUser ID: ' + self.myCore._crypto.pubKey)
+            logger.info(self.strings.programTag + '\n\nUser ID: ' + self.myCore._crypto.pubKey, terminal=True)
             if self.doSigs:
                 sigMsg = sigMsg % ('enabled',)
             else:
                 sigMsg = sigMsg % ('disabled (Your messages cannot be trusted)',)
             if self.doSigs:
-                logger.info(sigMsg)
+                logger.info(sigMsg, terminal=True)
             else:
-                logger.warn(sigMsg)
-            logger.info(self.strings.mainMenu.title()) # print out main menu
+                logger.warn(sigMsg, terminal=True)
+            logger.info(self.strings.mainMenu.title(), terminal=True) # print out main menu
             try:
                 choice = logger.readline('Enter 1-%s:\n' % (len(self.strings.mainMenuChoices))).lower().strip()
             except (KeyboardInterrupt, EOFError):
@@ -285,12 +285,12 @@ class OnionrMail:
             elif choice in (self.strings.mainMenuChoices[3], '4'):
                 self.toggle_signing()
             elif choice in (self.strings.mainMenuChoices[4], '5'):
-                logger.info('Goodbye.')
+                logger.info('Goodbye.', terminal=True)
                 break
             elif choice == '':
                 pass
             else:
-                logger.warn('Invalid choice.')
+                logger.warn('Invalid choice.', terminal=True)
         return
 
 def add_deleted(keyStore, bHash):

From 12227b6bcbd591c33fae02662894cc1ebad54178 Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Thu, 20 Jun 2019 02:59:32 -0500
Subject: [PATCH 04/29] a little work on clandestine UI

---
 .../default-plugins/clandestine/controlapi.py |  2 +-
 .../default-plugins/clandestine/main.py       |  2 +-
 .../default-plugins/clandestine/peerserver.py |  2 +-
 .../static-data/default-plugins/cliui/main.py |  2 +-
 .../default-plugins/contactmanager/main.py    |  2 +-
 .../default-plugins/flow/flowapi.py           |  2 +-
 .../static-data/default-plugins/flow/main.py  |  2 +-
 .../default-plugins/metadataprocessor/main.py |  2 +-
 .../default-plugins/pms/loadinbox.py          | 19 +++++++++++++++++++
 .../default-plugins/pms/mailapi.py            |  2 +-
 .../static-data/default-plugins/pms/main.py   |  2 +-
 .../default-plugins/pms/sentboxdb.py          |  2 +-
 onionr/static-data/www/clandestine/index.html |  1 +
 onionr/static-data/www/clandestine/js/main.js | 12 +++++++-----
 onionr/static-data/www/private/index.html     |  2 +-
 15 files changed, 39 insertions(+), 17 deletions(-)

diff --git a/onionr/static-data/default-plugins/clandestine/controlapi.py b/onionr/static-data/default-plugins/clandestine/controlapi.py
index 773e9c36..e40de9b6 100755
--- a/onionr/static-data/default-plugins/clandestine/controlapi.py
+++ b/onionr/static-data/default-plugins/clandestine/controlapi.py
@@ -1,5 +1,5 @@
 '''
-    Onionr - P2P Anonymous Storage Network
+    Onionr - Private P2P Communication
 
     HTTP endpoints for controlling IMs
 '''
diff --git a/onionr/static-data/default-plugins/clandestine/main.py b/onionr/static-data/default-plugins/clandestine/main.py
index 17007f0e..29afa3e1 100755
--- a/onionr/static-data/default-plugins/clandestine/main.py
+++ b/onionr/static-data/default-plugins/clandestine/main.py
@@ -1,5 +1,5 @@
 '''
-    Onionr - P2P Anonymous Storage Network
+    Onionr - Private P2P Communication
 
     Instant message conversations with Onionr peers
 '''
diff --git a/onionr/static-data/default-plugins/clandestine/peerserver.py b/onionr/static-data/default-plugins/clandestine/peerserver.py
index 5fe92fe9..5e4fba25 100755
--- a/onionr/static-data/default-plugins/clandestine/peerserver.py
+++ b/onionr/static-data/default-plugins/clandestine/peerserver.py
@@ -1,5 +1,5 @@
 '''
-    Onionr - P2P Anonymous Storage Network
+    Onionr - Private P2P Communication
 
     HTTP endpoints for communicating with peers
 '''
diff --git a/onionr/static-data/default-plugins/cliui/main.py b/onionr/static-data/default-plugins/cliui/main.py
index 5f47c455..f6a0b906 100755
--- a/onionr/static-data/default-plugins/cliui/main.py
+++ b/onionr/static-data/default-plugins/cliui/main.py
@@ -1,5 +1,5 @@
 '''
-    Onionr - P2P Anonymous Storage Network
+    Onionr - Private P2P Communication
 
     This is an interactive menu-driven CLI interface for Onionr
 '''
diff --git a/onionr/static-data/default-plugins/contactmanager/main.py b/onionr/static-data/default-plugins/contactmanager/main.py
index bdf6ac5b..d6c4499c 100755
--- a/onionr/static-data/default-plugins/contactmanager/main.py
+++ b/onionr/static-data/default-plugins/contactmanager/main.py
@@ -1,5 +1,5 @@
 '''
-    Onionr - P2P Anonymous Storage Network
+    Onionr - Private P2P Communication
 
     This is an interactive menu-driven CLI interface for Onionr
 '''
diff --git a/onionr/static-data/default-plugins/flow/flowapi.py b/onionr/static-data/default-plugins/flow/flowapi.py
index 95d5fb4f..09b54629 100755
--- a/onionr/static-data/default-plugins/flow/flowapi.py
+++ b/onionr/static-data/default-plugins/flow/flowapi.py
@@ -1,5 +1,5 @@
 '''
-    Onionr - P2P Microblogging Platform & Social network
+    Onionr - Private P2P Communication
 
     This file primarily serves to allow specific fetching of flow board messages
 '''
diff --git a/onionr/static-data/default-plugins/flow/main.py b/onionr/static-data/default-plugins/flow/main.py
index 0162019f..63c86cbc 100755
--- a/onionr/static-data/default-plugins/flow/main.py
+++ b/onionr/static-data/default-plugins/flow/main.py
@@ -1,5 +1,5 @@
 '''
-    Onionr - P2P Microblogging Platform & Social network
+    Onionr - Private P2P Communication
 
     This default plugin handles "flow" messages (global chatroom style communication)
 '''
diff --git a/onionr/static-data/default-plugins/metadataprocessor/main.py b/onionr/static-data/default-plugins/metadataprocessor/main.py
index 5feb4305..333c68aa 100755
--- a/onionr/static-data/default-plugins/metadataprocessor/main.py
+++ b/onionr/static-data/default-plugins/metadataprocessor/main.py
@@ -1,5 +1,5 @@
 '''
-    Onionr - P2P Anonymous Storage Network
+    Onionr - Private P2P Communication
 
     This processes metadata for Onionr blocks
 '''
diff --git a/onionr/static-data/default-plugins/pms/loadinbox.py b/onionr/static-data/default-plugins/pms/loadinbox.py
index 996d8f06..b4011b00 100755
--- a/onionr/static-data/default-plugins/pms/loadinbox.py
+++ b/onionr/static-data/default-plugins/pms/loadinbox.py
@@ -1,3 +1,22 @@
+'''
+    Onionr - Private P2P Communication
+
+    Load the user's inbox and return it as a list
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
 import onionrblockapi
 def load_inbox(myCore):
     inbox_list = []
diff --git a/onionr/static-data/default-plugins/pms/mailapi.py b/onionr/static-data/default-plugins/pms/mailapi.py
index 2ba99d08..af59bd93 100755
--- a/onionr/static-data/default-plugins/pms/mailapi.py
+++ b/onionr/static-data/default-plugins/pms/mailapi.py
@@ -1,5 +1,5 @@
 '''
-    Onionr - P2P Anonymous Storage Network
+    Onionr - Private P2P Communication
 
     HTTP endpoints for mail plugin.
 '''
diff --git a/onionr/static-data/default-plugins/pms/main.py b/onionr/static-data/default-plugins/pms/main.py
index e91a153c..e594c368 100755
--- a/onionr/static-data/default-plugins/pms/main.py
+++ b/onionr/static-data/default-plugins/pms/main.py
@@ -1,5 +1,5 @@
 '''
-    Onionr - P2P Anonymous Storage Network
+    Onionr - Private P2P Communication
 
     This default plugin handles private messages in an email like fashion
 '''
diff --git a/onionr/static-data/default-plugins/pms/sentboxdb.py b/onionr/static-data/default-plugins/pms/sentboxdb.py
index 28a5de6f..7090e214 100755
--- a/onionr/static-data/default-plugins/pms/sentboxdb.py
+++ b/onionr/static-data/default-plugins/pms/sentboxdb.py
@@ -1,5 +1,5 @@
 '''
-    Onionr - P2P Microblogging Platform & Social network
+    Onionr - Private P2P Communication
 
     This file handles the sentbox for the mail plugin
 '''
diff --git a/onionr/static-data/www/clandestine/index.html b/onionr/static-data/www/clandestine/index.html
index 91bc2e07..7fc0cab0 100755
--- a/onionr/static-data/www/clandestine/index.html
+++ b/onionr/static-data/www/clandestine/index.html
@@ -14,6 +14,7 @@
             <img class='logo' src='/shared/images/onionr-icon.png' alt='onionr logo'>
             <span class='logoText'>Clandestine</span>
 
+            <div>Current Used ID: <input class='myPub' type='text'></div>
             <ul class='conversationList'></ul>
         </div>
         <script src='/shared/misc.js'></script>
diff --git a/onionr/static-data/www/clandestine/js/main.js b/onionr/static-data/www/clandestine/js/main.js
index 0e7071a0..a757f713 100755
--- a/onionr/static-data/www/clandestine/js/main.js
+++ b/onionr/static-data/www/clandestine/js/main.js
@@ -1,12 +1,14 @@
-friendList = []
+friendList = {}
 convoListElement = document.getElementsByClassName('conversationList')[0]
 
 function createConvoList(){
-    for (var x = 0; x < friendList.length; x++){
+    console.log(friendList)
+
+    for (friend in friendList){
         var convoEntry = document.createElement('div')
         convoEntry.classList.add('convoEntry')
-        convoEntry.setAttribute('data-pubkey', friendList[x])
-        convoEntry.innerText = friendList[x]
+        convoEntry.setAttribute('data-pubkey', friend)
+        convoEntry.innerText = friendList[friend]
         convoListElement.append(convoEntry)
     }
 }
@@ -20,7 +22,7 @@ fetch('/friends/list', {
     var keys = []
     for(var k in resp) keys.push(k)
     for (var i = 0; i < keys.length; i++){
-        friendList.push(keys[i])
+        friendList[keys[i]] = resp[keys[i]]['name']
     }
     createConvoList()
 })
\ No newline at end of file
diff --git a/onionr/static-data/www/private/index.html b/onionr/static-data/www/private/index.html
index a2d3d3c9..9a548ebd 100755
--- a/onionr/static-data/www/private/index.html
+++ b/onionr/static-data/www/private/index.html
@@ -23,7 +23,7 @@
             <br>
             <button id='shutdownNode' class='btn warnBtn'>Shutdown Node</button> <button id='refreshStats' class='btn primaryBtn'>Refresh Stats</button>
             <br><br>
-            <label>Open Site: <input type='text' id='siteViewer' placeholder='Site Hash'> <button id='openSite' class='primaryBtn openSiteBtn'>Open Onionr Site</button></label>
+            <label>Open Site: <input type='text' id='siteViewer' placeholder='Site Hash'> <button id='openSite' class='btn primaryBtn openSiteBtn'>Open Onionr Site</button></label>
             <br>
             <br><br><a class='idLink' href='/mail/'>Mail</a> - <a class='idLink' href='/friends/'>Friend Manager</a> - <a class='idLink' href='/board/'>Circle</a> - 
                     <a class='idLink' href='/clandestine/'>Clandestine</a>

From 5cee375b028906b133c4519d0cdaff0ac914c297 Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Thu, 20 Jun 2019 16:59:36 -0500
Subject: [PATCH 05/29] fixed small cli logging bugs

---
 onionr/communicator.py | 2 +-
 onionr/onionr.py       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/onionr/communicator.py b/onionr/communicator.py
index 370a4981..d59b76da 100755
--- a/onionr/communicator.py
+++ b/onionr/communicator.py
@@ -317,7 +317,7 @@ class OnionrCommunicatorDaemon:
             try:
                 self.getPeerProfileInstance(peer).addScore(-10)
                 self.removeOnlinePeer(peer)
-                if action != 'ping':
+                if action != 'ping' and not self.shutdown:
                     logger.warn('Lost connection to ' + peer, terminal=True)
                     self.getOnlinePeers() # Will only add a new peer to pool if needed
             except ValueError:
diff --git a/onionr/onionr.py b/onionr/onionr.py
index e94e35b8..4030aae1 100755
--- a/onionr/onionr.py
+++ b/onionr/onionr.py
@@ -354,7 +354,7 @@ class Onionr:
             Displays a message suggesting help
         '''
         if __name__ == '__main__':
-            logger.info('Do ' + logger.colors.bold + sys.argv[0] + ' --help' + logger.colors.reset + logger.colors.fg.green + ' for Onionr help.')
+            logger.info('Do ' + logger.colors.bold + sys.argv[0] + ' --help' + logger.colors.reset + logger.colors.fg.green + ' for Onionr help.', terminal=True)
 
     def start(self, input = False, override = False):
         '''

From 50e93f46e40ba95f3535cbfb70f6c20256613c51 Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Sat, 22 Jun 2019 16:16:12 -0500
Subject: [PATCH 06/29] * Major core refactoring * renamed clandestine to
 esoteric

---
 onionr/communicatorutils/downloadblocks.py    |   2 +-
 onionr/core.py                                | 375 ++----------------
 onionr/coredb/__init__.py                     |   1 +
 onionr/coredb/blockmetadb/__init__.py         |  58 +++
 onionr/coredb/blockmetadb/expiredblocks.py    |  15 +
 onionr/coredb/blockmetadb/updateblockinfo.py  |  13 +
 onionr/coredb/daemonqueue/__init__.py         |  75 ++++
 onionr/coredb/keydb/__init__.py               |   1 +
 onionr/coredb/keydb/addkeys.py                |  70 ++++
 onionr/coredb/keydb/listkeys.py               |  63 +++
 onionr/coredb/keydb/removekeys.py             |  19 +
 onionr/coredb/keydb/transportinfo.py          |  53 +++
 onionr/coredb/keydb/userinfo.py               |  50 +++
 onionr/onionrstorage.py                       |   4 +-
 .../{clandestine => esoteric}/controlapi.py   |  12 +-
 .../{clandestine => esoteric}/info.json       |   2 +-
 .../{clandestine => esoteric}/main.py         |  10 +-
 .../{clandestine => esoteric}/peerserver.py   |  12 +-
 .../static-data/default-plugins/flow/main.py  |   1 +
 onionr/static-data/www/private/index.html     |   2 +-
 20 files changed, 472 insertions(+), 366 deletions(-)
 create mode 100644 onionr/coredb/__init__.py
 create mode 100644 onionr/coredb/blockmetadb/__init__.py
 create mode 100644 onionr/coredb/blockmetadb/expiredblocks.py
 create mode 100644 onionr/coredb/blockmetadb/updateblockinfo.py
 create mode 100644 onionr/coredb/daemonqueue/__init__.py
 create mode 100644 onionr/coredb/keydb/__init__.py
 create mode 100644 onionr/coredb/keydb/addkeys.py
 create mode 100644 onionr/coredb/keydb/listkeys.py
 create mode 100644 onionr/coredb/keydb/removekeys.py
 create mode 100644 onionr/coredb/keydb/transportinfo.py
 create mode 100644 onionr/coredb/keydb/userinfo.py
 rename onionr/static-data/default-plugins/{clandestine => esoteric}/controlapi.py (85%)
 rename onionr/static-data/default-plugins/{clandestine => esoteric}/info.json (64%)
 rename onionr/static-data/default-plugins/{clandestine => esoteric}/main.py (94%)
 rename onionr/static-data/default-plugins/{clandestine => esoteric}/peerserver.py (78%)

diff --git a/onionr/communicatorutils/downloadblocks.py b/onionr/communicatorutils/downloadblocks.py
index aad5f884..cb72bd12 100755
--- a/onionr/communicatorutils/downloadblocks.py
+++ b/onionr/communicatorutils/downloadblocks.py
@@ -110,7 +110,7 @@ def download_blocks_from_communicator(comm_inst):
             if removeFromQueue:
                 try:
                     del comm_inst.blockQueue[blockHash] # remove from block queue both if success or false
-                    logger.info('%s blocks remaining in queue' % [len(comm_inst.blockQueue)])
+                    logger.info('%s blocks remaining in queue' % [len(comm_inst.blockQueue)], terminal=True)
                 except KeyError:
                     pass
         comm_inst.currentDownloading.remove(blockHash)
diff --git a/onionr/core.py b/onionr/core.py
index be309309..735358e5 100755
--- a/onionr/core.py
+++ b/onionr/core.py
@@ -20,6 +20,7 @@
 import sqlite3, os, sys, time, json, uuid
 import logger, netcontroller, config
 from onionrblockapi import Block
+import coredb
 import deadsimplekv as simplekv
 import onionrutils, onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions
 import onionrblacklist
@@ -128,89 +129,19 @@ class Core:
         '''
             Adds a public key to the key database (misleading function name)
         '''
-        if peerID in self.listPeers() or peerID == self._crypto.pubKey:
-            raise ValueError("specified id is already known")
-
-        # This function simply adds a peer to the DB
-        if not self._utils.validatePubKey(peerID):
-            return False
-
-        events.event('pubkey_add', data = {'key': peerID}, onionr = self.onionrInst)
-
-        conn = sqlite3.connect(self.peerDB, timeout=30)
-        hashID = self._crypto.pubKeyHashID(peerID)
-        c = conn.cursor()
-        t = (peerID, name, 'unknown', hashID, 0)
-
-        for i in c.execute("SELECT * FROM peers WHERE id = ?;", (peerID,)):
-            try:
-                if i[0] == peerID:
-                    conn.close()
-                    return False
-            except ValueError:
-                pass
-            except IndexError:
-                pass
-        c.execute('INSERT INTO peers (id, name, dateSeen, hashID, trust) VALUES(?, ?, ?, ?, ?);', t)
-        conn.commit()
-        conn.close()
-
-        return True
+        return coredb.keydb.addkeys.add_peer(self, peerID, name)
 
     def addAddress(self, address):
         '''
             Add an address to the address database (only tor currently)
         '''
-
-        if type(address) is None or len(address) == 0:
-            return False
-        if self._utils.validateID(address):
-            if address == config.get('i2p.ownAddr', None) or address == self.hsAddress:
-                return False
-            conn = sqlite3.connect(self.addressDB, timeout=30)
-            c = conn.cursor()
-            # check if address is in database
-            # this is safe to do because the address is validated above, but we strip some chars here too just in case
-            address = address.replace('\'', '').replace(';', '').replace('"', '').replace('\\', '')
-            for i in c.execute("SELECT * FROM adders WHERE address = ?;", (address,)):
-                try:
-                    if i[0] == address:
-                        conn.close()
-                        return False
-                except ValueError:
-                    pass
-                except IndexError:
-                    pass
-
-            t = (address, 1)
-            c.execute('INSERT INTO adders (address, type) VALUES(?, ?);', t)
-            conn.commit()
-            conn.close()
-
-            events.event('address_add', data = {'address': address}, onionr = self.onionrInst)
-
-            return True
-        else:
-            #logger.debug('Invalid ID: %s' % address)
-            return False
+        return coredb.keydb.addkeys.add_address(self, address)
 
     def removeAddress(self, address):
         '''
             Remove an address from the address database
         '''
-
-        if self._utils.validateID(address):
-            conn = sqlite3.connect(self.addressDB, timeout=30)
-            c = conn.cursor()
-            t = (address,)
-            c.execute('Delete from adders where address=?;', t)
-            conn.commit()
-            conn.close()
-
-            events.event('address_remove', data = {'address': address}, onionr = self.onionrInst)
-            return True
-        else:
-            return False
+        return coredb.keydb.removekeys.remove_address(self, address)
 
     def removeBlock(self, block):
         '''
@@ -272,17 +203,6 @@ class Core:
         conn.commit()
         conn.close()
 
-        return
-
-    def getData(self, hash):
-        '''
-            Simply return the data associated to a hash
-        '''
-
-        data = onionrstorage.getData(self, hash)
-
-        return data
-
     def setData(self, data):
         '''
             Set the data assciated with a hash
@@ -299,10 +219,9 @@ class Core:
         if type(dataHash) is bytes:
             dataHash = dataHash.decode()
         blockFileName = self.blockDataLocation + dataHash + '.dat'
-        if os.path.exists(blockFileName):
-            pass # TODO: properly check if block is already saved elsewhere
-            #raise Exception("Data is already set for " + dataHash)
-        else:
+        try:
+            onionrstorage.getData(self, dataHash)
+        except onionrexceptions.NoDataAvailable:
             if self._utils.storageCounter.addBytes(dataSize) != False:
                 onionrstorage.store(self, data, blockHash=dataHash)
                 conn = sqlite3.connect(self.blockDB, timeout=30)
@@ -314,126 +233,48 @@ class Core:
                     nonceFile.write(dataHash + '\n')
             else:
                 raise onionrexceptions.DiskAllocationReached
+        else:
+            raise Exception("Data is already set for " + dataHash)
 
         return dataHash
 
+    def getData(self, hash):
+        '''
+            Simply return the data associated to a hash
+        '''
+        return onionrstorage.getData(self, hash)
+
     def daemonQueue(self):
         '''
             Gives commands to the communication proccess/daemon by reading an sqlite3 database
 
             This function intended to be used by the client. Queue to exchange data between "client" and server.
         '''
-
-        retData = False
-        if not os.path.exists(self.queueDB):
-            self.dbCreate.createDaemonDB()
-        else:
-            conn = sqlite3.connect(self.queueDB, timeout=30)
-            c = conn.cursor()
-            try:
-                for row in c.execute('SELECT command, data, date, min(ID), responseID FROM commands group by id'):
-                    retData = row
-                    break
-            except sqlite3.OperationalError:
-                self.dbCreate.createDaemonDB()
-            else:
-                if retData != False:
-                    c.execute('DELETE FROM commands WHERE id=?;', (retData[3],))
-            conn.commit()
-            conn.close()
-
-        events.event('queue_pop', data = {'data': retData}, onionr = self.onionrInst)
-
-        return retData
+        return coredb.daemonqueue.daemon_queue(self)
 
     def daemonQueueAdd(self, command, data='', responseID=''):
         '''
             Add a command to the daemon queue, used by the communication daemon (communicator.py)
         '''
-
-        retData = True
-
-        date = self._utils.getEpoch()
-        conn = sqlite3.connect(self.queueDB, timeout=30)
-        c = conn.cursor()
-        t = (command, data, date, responseID)
-        try:
-            c.execute('INSERT INTO commands (command, data, date, responseID) VALUES(?, ?, ?, ?)', t)
-            conn.commit()
-        except sqlite3.OperationalError:
-            retData = False
-            self.daemonQueue()
-        events.event('queue_push', data = {'command': command, 'data': data}, onionr = self.onionrInst)
-        conn.close()
-        return retData
+        return coredb.daemonqueue.daemon_queue_add(self, command, data, responseID)
 
     def daemonQueueGetResponse(self, responseID=''):
         '''
             Get a response sent by communicator to the API, by requesting to the API
         '''
-        assert len(responseID) > 0
-        resp = self._utils.localCommand('queueResponse/' + responseID)
-        return resp
-
-    def daemonQueueWaitForResponse(self, responseID='', checkFreqSecs=1):
-        resp = 'failure'
-        while resp == 'failure':
-            resp = self.daemonQueueGetResponse(responseID)
-            time.sleep(1)
-        return resp
-
-    def daemonQueueSimple(self, command, data='', checkFreqSecs=1):
-        '''
-        A simplified way to use the daemon queue. Will register a command (with optional data) and wait, return the data
-        Not always useful, but saves time + LOC in some cases.
-        This is a blocking function, so be careful.
-        '''
-        responseID = str(uuid.uuid4()) # generate unique response ID
-        self.daemonQueueAdd(command, data=data, responseID=responseID)
-        return self.daemonQueueWaitForResponse(responseID, checkFreqSecs)
+        return coredb.daemonqueue.daemon_queue_get_response(responseID)
 
     def clearDaemonQueue(self):
         '''
             Clear the daemon queue (somewhat dangerous)
         '''
-        conn = sqlite3.connect(self.queueDB, timeout=30)
-        c = conn.cursor()
-
-        try:
-            c.execute('DELETE FROM commands;')
-            conn.commit()
-        except:
-            pass
-
-        conn.close()
-        events.event('queue_clear', onionr = self.onionrInst)
-
-        return
+        return coredb.daemonqueue.clear_daemon_queue(self)
 
     def listAdders(self, randomOrder=True, i2p=True, recent=0):
         '''
             Return a list of addresses
         '''
-        conn = sqlite3.connect(self.addressDB, timeout=30)
-        c = conn.cursor()
-        if randomOrder:
-            addresses = c.execute('SELECT * FROM adders ORDER BY RANDOM();')
-        else:
-            addresses = c.execute('SELECT * FROM adders;')
-        addressList = []
-        for i in addresses:
-            if len(i[0].strip()) == 0:
-                continue
-            addressList.append(i[0])
-        conn.close()
-        testList = list(addressList) # create new list to iterate
-        for address in testList:
-            try:
-                if recent > 0 and (self._utils.getEpoch() - self.getAddressInfo(address, 'lastConnect')) > recent:
-                    raise TypeError # If there is no last-connected date or it was too long ago, don't add peer to list if recent is not 0
-            except TypeError:
-                addressList.remove(address)
-        return addressList
+        return coredb.keydb.listkeys.list_adders(self, randomOrder, i2p, recent)
 
     def listPeers(self, randomOrder=True, getPow=False, trust=0):
         '''
@@ -442,35 +283,7 @@ class Core:
             randomOrder determines if the list should be in a random order
             trust sets the minimum trust to list
         '''
-        conn = sqlite3.connect(self.peerDB, timeout=30)
-        c = conn.cursor()
-
-        payload = ''
-
-        if trust not in (0, 1, 2):
-            logger.error('Tried to select invalid trust.')
-            return
-
-        if randomOrder:
-            payload = 'SELECT * FROM peers WHERE trust >= ? ORDER BY RANDOM();'
-        else:
-            payload = 'SELECT * FROM peers WHERE trust >= ?;'
-
-        peerList = []
-
-        for i in c.execute(payload, (trust,)):
-            try:
-                if len(i[0]) != 0:
-                    if getPow:
-                        peerList.append(i[0] + '-' + i[1])
-                    else:
-                        peerList.append(i[0])
-            except TypeError:
-                pass
-
-        conn.close()
-
-        return peerList
+        return coredb.keydb.listkeys.list_peers(self, randomOrder, getPow, trust)
 
     def getPeerInfo(self, peer, info):
         '''
@@ -483,46 +296,13 @@ class Core:
             trust int           4
             hashID text         5
         '''
-        conn = sqlite3.connect(self.peerDB, timeout=30)
-        c = conn.cursor()
-
-        command = (peer,)
-        infoNumbers = {'id': 0, 'name': 1, 'adders': 2, 'dateSeen': 3, 'trust': 4, 'hashID': 5}
-        info = infoNumbers[info]
-        iterCount = 0
-        retVal = ''
-
-        for row in c.execute('SELECT * FROM peers WHERE id=?;', command):
-            for i in row:
-                if iterCount == info:
-                    retVal = i
-                    break
-                else:
-                    iterCount += 1
-
-        conn.close()
-
-        return retVal
+        return coredb.keydb.userinfo.get_user_info(self, peer, info)
 
     def setPeerInfo(self, peer, key, data):
         '''
             Update a peer for a key
         '''
-
-        conn = sqlite3.connect(self.peerDB, timeout=30)
-        c = conn.cursor()
-
-        command = (data, peer)
-
-        # TODO: validate key on whitelist
-        if key not in ('id', 'name', 'pubkey', 'forwardKey', 'dateSeen', 'trust'):
-            raise Exception("Got invalid database key when setting peer info")
-
-        c.execute('UPDATE peers SET ' + key + ' = ? WHERE id=?', command)
-        conn.commit()
-        conn.close()
-
-        return
+        return coredb.keydb.userinfo.set_peer_info(self, peer, key, data)
 
     def getAddressInfo(self, address, info):
         '''
@@ -539,117 +319,35 @@ class Core:
             trust       8
             introduced  9
         '''
-
-        conn = sqlite3.connect(self.addressDB, timeout=30)
-        c = conn.cursor()
-
-        command = (address,)
-        infoNumbers = {'address': 0, 'type': 1, 'knownPeer': 2, 'speed': 3, 'success': 4, 'powValue': 5, 'failure': 6, 'lastConnect': 7, 'trust': 8, 'introduced': 9}
-        info = infoNumbers[info]
-        iterCount = 0
-        retVal = ''
-
-        for row in c.execute('SELECT * FROM adders WHERE address=?;', command):
-            for i in row:
-                if iterCount == info:
-                    retVal = i
-                    break
-                else:
-                    iterCount += 1
-        conn.close()
-
-        return retVal
+        return coredb.keydb.transportinfo.get_address_info(self, address, info)
 
     def setAddressInfo(self, address, key, data):
         '''
             Update an address for a key
         '''
-
-        conn = sqlite3.connect(self.addressDB, timeout=30)
-        c = conn.cursor()
-
-        command = (data, address)
-
-        if key not in ('address', 'type', 'knownPeer', 'speed', 'success', 'failure', 'powValue', 'lastConnect', 'lastConnectAttempt', 'trust', 'introduced'):
-            raise Exception("Got invalid database key when setting address info")
-        else:
-            c.execute('UPDATE adders SET ' + key + ' = ? WHERE address=?', command)
-            conn.commit()
-        conn.close()
-
-        return
+        return coredb.keydb.transportinfo.set_address_info(self, address, key, data)
 
     def getBlockList(self, dateRec = None, unsaved = False):
         '''
             Get list of our blocks
         '''
-        if dateRec == None:
-            dateRec = 0
-
-        conn = sqlite3.connect(self.blockDB, timeout=30)
-        c = conn.cursor()
-
-        execute = 'SELECT hash FROM hashes WHERE dateReceived >= ? ORDER BY dateReceived ASC;'
-        args = (dateRec,)
-        rows = list()
-        for row in c.execute(execute, args):
-            for i in row:
-                rows.append(i)
-        conn.close()
-        return rows
+        return coredb.blockmetadb.get_block_list(self, dateRec, unsaved)
 
     def getBlockDate(self, blockHash):
         '''
             Returns the date a block was received
         '''
-
-        conn = sqlite3.connect(self.blockDB, timeout=30)
-        c = conn.cursor()
-
-        execute = 'SELECT dateReceived FROM hashes WHERE hash=?;'
-        args = (blockHash,)
-        for row in c.execute(execute, args):
-            for i in row:
-                return int(i)
-        conn.close()
-        return None
+        return coredb.blockmetadb.get_block_date(self, blockHash)
 
     def getBlocksByType(self, blockType, orderDate=True):
         '''
             Returns a list of blocks by the type
         '''
-
-        conn = sqlite3.connect(self.blockDB, timeout=30)
-        c = conn.cursor()
-
-        if orderDate:
-            execute = 'SELECT hash FROM hashes WHERE dataType=? ORDER BY dateReceived;'
-        else:
-            execute = 'SELECT hash FROM hashes WHERE dataType=?;'
-
-        args = (blockType,)
-        rows = list()
-
-        for row in c.execute(execute, args):
-            for i in row:
-                rows.append(i)
-        conn.close()
-        return rows
+        return coredb.blockmetadb.get_blocks_by_type(self, blockType, orderDate)
 
     def getExpiredBlocks(self):
         '''Returns a list of expired blocks'''
-        conn = sqlite3.connect(self.blockDB, timeout=30)
-        c = conn.cursor()
-        date = int(self._utils.getEpoch())
-
-        execute = 'SELECT hash FROM hashes WHERE expire <= %s ORDER BY dateReceived;' % (date,)
-
-        rows = list()
-        for row in c.execute(execute):
-            for i in row:
-                rows.append(i)
-        conn.close()
-        return rows
+        return coredb.blockmetadb.expiredblocks.get_expired_blocks(self)
 
     def updateBlockInfo(self, hash, key, data):
         '''
@@ -666,18 +364,7 @@ class Core:
             dateClaimed  - timestamp claimed inside the block, only as trustworthy as the block author is
             expire       - expire date for a block
         '''
-
-        if key not in ('dateReceived', 'decrypted', 'dataType', 'dataFound', 'dataSaved', 'sig', 'author', 'dateClaimed', 'expire'):
-            return False
-
-        conn = sqlite3.connect(self.blockDB, timeout=30)
-        c = conn.cursor()
-        args = (data, hash)
-        c.execute("UPDATE hashes SET " + key + " = ? where hash = ?;", args)
-        conn.commit()
-        conn.close()
-
-        return True
+        return coredb.blockmetadb.updateblockinfo
 
     def insertBlock(self, data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = {}, expire=None, disableForward=False):
         '''
@@ -695,8 +382,6 @@ class Core:
 
         createTime = self._utils.getRoundedEpoch()
 
-        # check nonce
-        #print(data)
         dataNonce = self._utils.bytesToStr(self._crypto.sha3Hash(data))
         try:
             with open(self.dataNonceFile, 'r') as nonces:
diff --git a/onionr/coredb/__init__.py b/onionr/coredb/__init__.py
new file mode 100644
index 00000000..39c909ba
--- /dev/null
+++ b/onionr/coredb/__init__.py
@@ -0,0 +1 @@
+from . import keydb, blockmetadb, daemonqueue
\ No newline at end of file
diff --git a/onionr/coredb/blockmetadb/__init__.py b/onionr/coredb/blockmetadb/__init__.py
new file mode 100644
index 00000000..fd408d35
--- /dev/null
+++ b/onionr/coredb/blockmetadb/__init__.py
@@ -0,0 +1,58 @@
+import sqlite3
+from . import expiredblocks, updateblockinfo
+def get_block_list(core_inst, dateRec = None, unsaved = False):
+    '''
+        Get list of our blocks
+    '''
+    if dateRec == None:
+        dateRec = 0
+
+    conn = sqlite3.connect(core_inst.blockDB, timeout=30)
+    c = conn.cursor()
+
+    execute = 'SELECT hash FROM hashes WHERE dateReceived >= ? ORDER BY dateReceived ASC;'
+    args = (dateRec,)
+    rows = list()
+    for row in c.execute(execute, args):
+        for i in row:
+            rows.append(i)
+    conn.close()
+    return rows
+
+def get_block_date(core_inst, blockHash):
+    '''
+        Returns the date a block was received
+    '''
+
+    conn = sqlite3.connect(core_inst.blockDB, timeout=30)
+    c = conn.cursor()
+
+    execute = 'SELECT dateReceived FROM hashes WHERE hash=?;'
+    args = (blockHash,)
+    for row in c.execute(execute, args):
+        for i in row:
+            return int(i)
+    conn.close()
+    return None
+
+def get_blocks_by_type(core_inst, blockType, orderDate=True):
+    '''
+        Returns a list of blocks by the type
+    '''
+
+    conn = sqlite3.connect(core_inst.blockDB, timeout=30)
+    c = conn.cursor()
+
+    if orderDate:
+        execute = 'SELECT hash FROM hashes WHERE dataType=? ORDER BY dateReceived;'
+    else:
+        execute = 'SELECT hash FROM hashes WHERE dataType=?;'
+
+    args = (blockType,)
+    rows = list()
+
+    for row in c.execute(execute, args):
+        for i in row:
+            rows.append(i)
+    conn.close()
+    return rows
\ No newline at end of file
diff --git a/onionr/coredb/blockmetadb/expiredblocks.py b/onionr/coredb/blockmetadb/expiredblocks.py
new file mode 100644
index 00000000..8debbe26
--- /dev/null
+++ b/onionr/coredb/blockmetadb/expiredblocks.py
@@ -0,0 +1,15 @@
+import sqlite3
+def get_expired_blocks(core_inst):
+    '''Returns a list of expired blocks'''
+    conn = sqlite3.connect(core_inst.blockDB, timeout=30)
+    c = conn.cursor()
+    date = int(core_inst._utils.getEpoch())
+
+    execute = 'SELECT hash FROM hashes WHERE expire <= %s ORDER BY dateReceived;' % (date,)
+
+    rows = list()
+    for row in c.execute(execute):
+        for i in row:
+            rows.append(i)
+    conn.close()
+    return rows
\ No newline at end of file
diff --git a/onionr/coredb/blockmetadb/updateblockinfo.py b/onionr/coredb/blockmetadb/updateblockinfo.py
new file mode 100644
index 00000000..05f0fc8a
--- /dev/null
+++ b/onionr/coredb/blockmetadb/updateblockinfo.py
@@ -0,0 +1,13 @@
+import sqlite3
+def update_block_info(core_inst, hash, key, data):
+    if key not in ('dateReceived', 'decrypted', 'dataType', 'dataFound', 'dataSaved', 'sig', 'author', 'dateClaimed', 'expire'):
+        return False
+
+    conn = sqlite3.connect(core_inst.blockDB, timeout=30)
+    c = conn.cursor()
+    args = (data, hash)
+    c.execute("UPDATE hashes SET " + key + " = ? where hash = ?;", args)
+    conn.commit()
+    conn.close()
+
+    return True
\ No newline at end of file
diff --git a/onionr/coredb/daemonqueue/__init__.py b/onionr/coredb/daemonqueue/__init__.py
new file mode 100644
index 00000000..8bd21cfd
--- /dev/null
+++ b/onionr/coredb/daemonqueue/__init__.py
@@ -0,0 +1,75 @@
+import sqlite3, os
+import onionrevents as events
+def daemon_queue(core_inst):
+    '''
+        Gives commands to the communication proccess/daemon by reading an sqlite3 database
+
+        This function intended to be used by the client. Queue to exchange data between "client" and server.
+    '''
+
+    retData = False
+    if not os.path.exists(core_inst.queueDB):
+        core_inst.dbCreate.createDaemonDB()
+    else:
+        conn = sqlite3.connect(core_inst.queueDB, timeout=30)
+        c = conn.cursor()
+        try:
+            for row in c.execute('SELECT command, data, date, min(ID), responseID FROM commands group by id'):
+                retData = row
+                break
+        except sqlite3.OperationalError:
+            core_inst.dbCreate.createDaemonDB()
+        else:
+            if retData != False:
+                c.execute('DELETE FROM commands WHERE id=?;', (retData[3],))
+        conn.commit()
+        conn.close()
+
+    events.event('queue_pop', data = {'data': retData}, onionr = core_inst.onionrInst)
+
+    return retData
+
+def daemon_queue_add(core_inst, command, data='', responseID=''):
+    '''
+        Add a command to the daemon queue, used by the communication daemon (communicator.py)
+    '''
+
+    retData = True
+
+    date = core_inst._utils.getEpoch()
+    conn = sqlite3.connect(core_inst.queueDB, timeout=30)
+    c = conn.cursor()
+    t = (command, data, date, responseID)
+    try:
+        c.execute('INSERT INTO commands (command, data, date, responseID) VALUES(?, ?, ?, ?)', t)
+        conn.commit()
+    except sqlite3.OperationalError:
+        retData = False
+        core_inst.daemonQueue()
+    events.event('queue_push', data = {'command': command, 'data': data}, onionr = core_inst.onionrInst)
+    conn.close()
+    return retData
+
+def daemon_queue_get_response(core_inst, responseID=''):
+    '''
+        Get a response sent by communicator to the API, by requesting to the API
+    '''
+    assert len(responseID) > 0
+    resp = core_inst._utils.localCommand('queueResponse/' + responseID)
+    return resp
+
+def clear_daemon_queue(core_inst):
+    '''
+        Clear the daemon queue (somewhat dangerous)
+    '''
+    conn = sqlite3.connect(core_inst.queueDB, timeout=30)
+    c = conn.cursor()
+
+    try:
+        c.execute('DELETE FROM commands;')
+        conn.commit()
+    except:
+        pass
+
+    conn.close()
+    events.event('queue_clear', onionr = core_inst.onionrInst)
\ No newline at end of file
diff --git a/onionr/coredb/keydb/__init__.py b/onionr/coredb/keydb/__init__.py
new file mode 100644
index 00000000..16d602d5
--- /dev/null
+++ b/onionr/coredb/keydb/__init__.py
@@ -0,0 +1 @@
+from . import addkeys, listkeys, removekeys, userinfo, transportinfo
\ No newline at end of file
diff --git a/onionr/coredb/keydb/addkeys.py b/onionr/coredb/keydb/addkeys.py
new file mode 100644
index 00000000..ac219b1a
--- /dev/null
+++ b/onionr/coredb/keydb/addkeys.py
@@ -0,0 +1,70 @@
+import sqlite3
+import onionrevents as events, config
+def add_peer(core_inst, peerID, name=''):
+    '''
+        Adds a public key to the key database (misleading function name)
+    '''
+    if peerID in core_inst.listPeers() or peerID == core_inst._crypto.pubKey:
+        raise ValueError("specified id is already known")
+
+    # This function simply adds a peer to the DB
+    if not core_inst._utils.validatePubKey(peerID):
+        return False
+
+    events.event('pubkey_add', data = {'key': peerID}, onionr = core_inst.onionrInst)
+
+    conn = sqlite3.connect(core_inst.peerDB, timeout=30)
+    hashID = core_inst._crypto.pubKeyHashID(peerID)
+    c = conn.cursor()
+    t = (peerID, name, 'unknown', hashID, 0)
+
+    for i in c.execute("SELECT * FROM peers WHERE id = ?;", (peerID,)):
+        try:
+            if i[0] == peerID:
+                conn.close()
+                return False
+        except ValueError:
+            pass
+        except IndexError:
+            pass
+    c.execute('INSERT INTO peers (id, name, dateSeen, hashID, trust) VALUES(?, ?, ?, ?, ?);', t)
+    conn.commit()
+    conn.close()
+
+    return True
+
+def add_address(core_inst, address):
+    '''
+        Add an address to the address database (only tor currently)
+    '''
+
+    if type(address) is None or len(address) == 0:
+        return False
+    if core_inst._utils.validateID(address):
+        if address == config.get('i2p.ownAddr', None) or address == core_inst.hsAddress:
+            return False
+        conn = sqlite3.connect(core_inst.addressDB, timeout=30)
+        c = conn.cursor()
+        # check if address is in database
+        # this is safe to do because the address is validated above, but we strip some chars here too just in case
+        address = address.replace('\'', '').replace(';', '').replace('"', '').replace('\\', '')
+        for i in c.execute("SELECT * FROM adders WHERE address = ?;", (address,)):
+            try:
+                if i[0] == address:
+                    conn.close()
+                    return False
+            except ValueError:
+                pass
+            except IndexError:
+                pass
+
+        t = (address, 1)
+        c.execute('INSERT INTO adders (address, type) VALUES(?, ?);', t)
+        conn.commit()
+        conn.close()
+
+        events.event('address_add', data = {'address': address}, onionr = core_inst.onionrInst)
+
+        return True
+    else:
+        return False
diff --git a/onionr/coredb/keydb/listkeys.py b/onionr/coredb/keydb/listkeys.py
new file mode 100644
index 00000000..5ab44fb9
--- /dev/null
+++ b/onionr/coredb/keydb/listkeys.py
@@ -0,0 +1,63 @@
+import sqlite3
+import logger
+def list_peers(core_inst, randomOrder=True, getPow=False, trust=0):
+    '''
+        Return a list of public keys (misleading function name)
+
+        randomOrder determines if the list should be in a random order
+        trust sets the minimum trust to list
+    '''
+    conn = sqlite3.connect(core_inst.peerDB, timeout=30)
+    c = conn.cursor()
+
+    payload = ''
+
+    if trust not in (0, 1, 2):
+        logger.error('Tried to select invalid trust.')
+        return
+
+    if randomOrder:
+        payload = 'SELECT * FROM peers WHERE trust >= ? ORDER BY RANDOM();'
+    else:
+        payload = 'SELECT * FROM peers WHERE trust >= ?;'
+
+    peerList = []
+
+    for i in c.execute(payload, (trust,)):
+        try:
+            if len(i[0]) != 0:
+                if getPow:
+                    peerList.append(i[0] + '-' + i[1])
+                else:
+                    peerList.append(i[0])
+        except TypeError:
+            pass
+
+    conn.close()
+
+    return peerList
+
+def list_adders(core_inst, randomOrder=True, i2p=True, recent=0):
+    '''
+        Return a list of transport addresses
+    '''
+    conn = sqlite3.connect(core_inst.addressDB, timeout=30)
+    c = conn.cursor()
+    if randomOrder:
+        addresses = c.execute('SELECT * FROM adders ORDER BY RANDOM();')
+    else:
+        addresses = c.execute('SELECT * FROM adders;')
+    addressList = []
+    for i in addresses:
+        if len(i[0].strip()) == 0:
+            continue
+        addressList.append(i[0])
+    conn.close()
+    testList = list(addressList) # create new list to iterate
+    for address in testList:
+        try:
+            if recent > 0 and (core_inst._utils.getEpoch() - core_inst.getAddressInfo(address, 'lastConnect')) > recent:
+                raise TypeError # If there is no last-connected date or it was too long ago, don't add peer to list if recent is not 0
+        except TypeError:
+            addressList.remove(address)
+    return addressList
\ No newline at end of file
diff --git a/onionr/coredb/keydb/removekeys.py b/onionr/coredb/keydb/removekeys.py
new file mode 100644
index 00000000..10f44a1b
--- /dev/null
+++ b/onionr/coredb/keydb/removekeys.py
@@ -0,0 +1,19 @@
+import sqlite3
+import onionrevents as events
+def remove_address(core_inst, address):
+    '''
+        Remove an address from the address database
+    '''
+
+    if core_inst._utils.validateID(address):
+        conn = sqlite3.connect(core_inst.addressDB, timeout=30)
+        c = conn.cursor()
+        t = (address,)
+        c.execute('Delete from adders where address=?;', t)
+        conn.commit()
+        conn.close()
+
+        events.event('address_remove', data = {'address': address}, onionr = core_inst.onionrInst)
+        return True
+    else:
+        return False
\ No newline at end of file
diff --git a/onionr/coredb/keydb/transportinfo.py b/onionr/coredb/keydb/transportinfo.py
new file mode 100644
index 00000000..9fd642ec
--- /dev/null
+++ b/onionr/coredb/keydb/transportinfo.py
@@ -0,0 +1,53 @@
+import sqlite3
+def get_address_info(core_inst, address, info):
+    '''
+        Get info about an address from its database entry
+
+        address text, 0
+        type int, 1
+        knownPeer text, 2
+        speed int, 3
+        success int, 4
+        powValue    5
+        failure int 6
+        lastConnect 7
+        trust       8
+        introduced  9
+    '''
+
+    conn = sqlite3.connect(core_inst.addressDB, timeout=30)
+    c = conn.cursor()
+
+    command = (address,)
+    infoNumbers = {'address': 0, 'type': 1, 'knownPeer': 2, 'speed': 3, 'success': 4, 'powValue': 5, 'failure': 6, 'lastConnect': 7, 'trust': 8, 'introduced': 9}
+    info = infoNumbers[info]
+    iterCount = 0
+    retVal = ''
+
+    for row in c.execute('SELECT * FROM adders WHERE address=?;', command):
+        for i in row:
+            if iterCount == info:
+                retVal = i
+                break
+            else:
+                iterCount += 1
+    conn.close()
+
+    return retVal
+
+def set_address_info(core_inst, address, key, data):
+    '''
+        Update an address for a key
+    '''
+
+    conn = sqlite3.connect(core_inst.addressDB, timeout=30)
+    c = conn.cursor()
+
+    command = (data, address)
+
+    if key not in ('address', 'type', 'knownPeer', 'speed', 'success', 'failure', 'powValue', 'lastConnect', 'lastConnectAttempt', 'trust', 'introduced'):
+        raise Exception("Got invalid database key when setting address info")
+    else:
+        c.execute('UPDATE adders SET ' + key + ' = ? WHERE address=?', command)
+        conn.commit()
+    conn.close()
\ No newline at end of file
diff --git a/onionr/coredb/keydb/userinfo.py b/onionr/coredb/keydb/userinfo.py
new file mode 100644
index 00000000..84a737d6
--- /dev/null
+++ b/onionr/coredb/keydb/userinfo.py
@@ -0,0 +1,50 @@
+import sqlite3
+def get_user_info(core_inst, peer, info):
+    '''
+        Get info about a peer from their database entry
+
+        id text             0
+        name text,          1
+        adders text,        2
+        dateSeen not null,  3
+        trust int           4
+        hashID text         5
+    '''
+    conn = sqlite3.connect(core_inst.peerDB, timeout=30)
+    c = conn.cursor()
+
+    command = (peer,)
+    infoNumbers = {'id': 0, 'name': 1, 'adders': 2, 'dateSeen': 3, 'trust': 4, 'hashID': 5}
+    info = infoNumbers[info]
+    iterCount = 0
+    retVal = ''
+
+    for row in c.execute('SELECT * FROM peers WHERE id=?;', command):
+        for i in row:
+            if iterCount == info:
+                retVal = i
+                break
+            else:
+                iterCount += 1
+
+    conn.close()
+
+    return retVal
+
+def set_peer_info(core_inst, peer, key, data):
+    '''
+        Update a peer for a key
+    '''
+
+    conn = sqlite3.connect(core_inst.peerDB, timeout=30)
+    c = conn.cursor()
+
+    command = (data, peer)
+
+    # TODO: validate key on whitelist
+    if key not in ('id', 'name', 'pubkey', 'forwardKey', 'dateSeen', 'trust'):
+        raise Exception("Got invalid database key when setting peer info")
+
+    c.execute('UPDATE peers SET ' + key + ' = ? WHERE id=?', command)
+    conn.commit()
+    conn.close()
\ No newline at end of file
diff --git a/onionr/onionrstorage.py b/onionr/onionrstorage.py
index 5f44c4f3..b859662d 100755
--- a/onionr/onionrstorage.py
+++ b/onionr/onionrstorage.py
@@ -17,7 +17,7 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import core, sys, sqlite3, os, dbcreator
+import core, sys, sqlite3, os, dbcreator, onionrexceptions
 
 DB_ENTRY_SIZE_LIMIT = 10000 # Will be a config option
 
@@ -94,4 +94,6 @@ def getData(coreInst, bHash):
             retData = block.read()
     else:
         retData = _dbFetch(coreInst, bHash)
+        if retData is None:
+            raise onionrexceptions.NoDataAvailable("Block data for %s is not available" % [bHash])
     return retData
\ No newline at end of file
diff --git a/onionr/static-data/default-plugins/clandestine/controlapi.py b/onionr/static-data/default-plugins/esoteric/controlapi.py
similarity index 85%
rename from onionr/static-data/default-plugins/clandestine/controlapi.py
rename to onionr/static-data/default-plugins/esoteric/controlapi.py
index e40de9b6..7c298063 100755
--- a/onionr/static-data/default-plugins/clandestine/controlapi.py
+++ b/onionr/static-data/default-plugins/esoteric/controlapi.py
@@ -22,13 +22,13 @@ from flask import Response, request, redirect, Blueprint, send_from_directory
 import core
 
 core_inst = core.Core()
-flask_blueprint = Blueprint('clandestine_control', __name__)
+flask_blueprint = Blueprint('esoteric_control', __name__)
 
-@flask_blueprint.route('/clandestine/ping')
+@flask_blueprint.route('/esoteric/ping')
 def ping():
     return 'pong!'
 
-@flask_blueprint.route('/clandestine/send/<peer>', methods=['POST'])
+@flask_blueprint.route('/esoteric/send/<peer>', methods=['POST'])
 def send_message(peer):
     data = request.get_json(force=True)
     core_inst.keyStore.refresh()
@@ -40,14 +40,14 @@ def send_message(peer):
     core_inst.keyStore.flush()
     return Response('success')
 
-@flask_blueprint.route('/clandestine/gets/<peer>')
+@flask_blueprint.route('/esoteric/gets/<peer>')
 def get_sent(peer):
     sent = core_inst.keyStore.get('s' + peer)
     if sent is None:
         sent = []
     return Response(json.dumps(sent))
 
-@flask_blueprint.route('/clandestine/addrec/<peer>', methods=['POST'])
+@flask_blueprint.route('/esoteric/addrec/<peer>', methods=['POST'])
 def add_rec(peer):
     data = request.get_json(force=True)
     core_inst.keyStore.refresh()
@@ -59,7 +59,7 @@ def add_rec(peer):
     core_inst.keyStore.flush()
     return Response('success')
 
-@flask_blueprint.route('/clandestine/getrec/<peer>')
+@flask_blueprint.route('/esoteric/getrec/<peer>')
 def get_messages(peer):
     core_inst.keyStore.refresh()
     existing = core_inst.keyStore.get('r' + peer)
diff --git a/onionr/static-data/default-plugins/clandestine/info.json b/onionr/static-data/default-plugins/esoteric/info.json
similarity index 64%
rename from onionr/static-data/default-plugins/clandestine/info.json
rename to onionr/static-data/default-plugins/esoteric/info.json
index 7c5a143c..6fcc6e0e 100755
--- a/onionr/static-data/default-plugins/clandestine/info.json
+++ b/onionr/static-data/default-plugins/esoteric/info.json
@@ -1,5 +1,5 @@
 {
-    "name" : "clandestine",
+    "name" : "esoteric",
     "version" : "1.0",
     "author" : "onionr"
 }
diff --git a/onionr/static-data/default-plugins/clandestine/main.py b/onionr/static-data/default-plugins/esoteric/main.py
similarity index 94%
rename from onionr/static-data/default-plugins/clandestine/main.py
rename to onionr/static-data/default-plugins/esoteric/main.py
index 29afa3e1..1f4cdd32 100755
--- a/onionr/static-data/default-plugins/clandestine/main.py
+++ b/onionr/static-data/default-plugins/esoteric/main.py
@@ -24,7 +24,7 @@ locale.setlocale(locale.LC_ALL, '')
 import onionrservices, logger
 from onionrservices import bootstrapservice
 
-plugin_name = 'clandestine'
+plugin_name = 'esoteric'
 PLUGIN_VERSION = '0.0.0'
 sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
 import controlapi, peerserver
@@ -36,7 +36,7 @@ def exit_with_error(text=''):
         logger.error(text)
     sys.exit(1)
 
-class Clandestine:
+class Esoteric:
     def __init__(self, pluginapi):
         self.myCore = pluginapi.get_core()
         self.peer = None
@@ -58,7 +58,7 @@ class Clandestine:
                     message += '\n'
             except EOFError:
                 message = json.dumps({'m': message, 't': self.myCore._utils.getEpoch()})
-                print(self.myCore._utils.doPostRequest('http://%s/clandestine/sendto' % (self.transport,), port=self.socks, data=message))
+                print(self.myCore._utils.doPostRequest('http://%s/esoteric/sendto' % (self.transport,), port=self.socks, data=message))
                 message = ''
             except KeyboardInterrupt:
                 self.shutdown = True
@@ -89,6 +89,6 @@ def on_init(api, data = None):
     '''
 
     pluginapi = api
-    chat = Clandestine(pluginapi)
-    api.commands.register(['clandestine'], chat.create)
+    chat = Esoteric(pluginapi)
+    api.commands.register(['esoteric'], chat.create)
     return
diff --git a/onionr/static-data/default-plugins/clandestine/peerserver.py b/onionr/static-data/default-plugins/esoteric/peerserver.py
similarity index 78%
rename from onionr/static-data/default-plugins/clandestine/peerserver.py
rename to onionr/static-data/default-plugins/esoteric/peerserver.py
index 5e4fba25..b3ae423e 100755
--- a/onionr/static-data/default-plugins/clandestine/peerserver.py
+++ b/onionr/static-data/default-plugins/esoteric/peerserver.py
@@ -21,7 +21,7 @@ import sys, os, json
 import core
 from flask import Response, request, redirect, Blueprint, abort, g
 sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
-direct_blueprint = Blueprint('clandestine', __name__)
+direct_blueprint = Blueprint('esoteric', __name__)
 core_inst = core.Core()
 
 storage_dir = core_inst.dataDir
@@ -35,11 +35,11 @@ def request_setup():
     g.host = host
     g.peer = core_inst.keyStore.get('dc-' + g.host)
 
-@direct_blueprint.route('/clandestine/ping')
+@direct_blueprint.route('/esoteric/ping')
 def pingdirect():
     return 'pong!'
 
-@direct_blueprint.route('/clandestine/sendto', methods=['POST', 'GET'])
+@direct_blueprint.route('/esoteric/sendto', methods=['POST', 'GET'])
 def sendto():
     try:
         msg = request.get_json(force=True)
@@ -47,9 +47,9 @@ def sendto():
         msg = ''
     else:
         msg = json.dumps(msg)
-        core_inst._utils.localCommand('/clandestine/addrec/%s' % (g.peer,), post=True, postData=msg)
+        core_inst._utils.localCommand('/esoteric/addrec/%s' % (g.peer,), post=True, postData=msg)
     return Response('success')
 
-@direct_blueprint.route('/clandestine/poll')
+@direct_blueprint.route('/esoteric/poll')
 def poll_chat():
-    return Response(core_inst._utils.localCommand('/clandestine/gets/%s' % (g.peer,)))
\ No newline at end of file
+    return Response(core_inst._utils.localCommand('/esoteric/gets/%s' % (g.peer,)))
\ No newline at end of file
diff --git a/onionr/static-data/default-plugins/flow/main.py b/onionr/static-data/default-plugins/flow/main.py
index 63c86cbc..d1c2c87d 100755
--- a/onionr/static-data/default-plugins/flow/main.py
+++ b/onionr/static-data/default-plugins/flow/main.py
@@ -61,6 +61,7 @@ class OnionrFlow:
                     self.flowRunning = False
                 expireTime = self.myCore._utils.getEpoch() + 43200
                 if len(message) > 0:
+                    logger.info('Inserting message as block...', terminal=True)
                     self.myCore.insertBlock(message, header='txt', expire=expireTime, meta={'ch': self.channel})
 
         logger.info("Flow is exiting, goodbye", terminal=True)
diff --git a/onionr/static-data/www/private/index.html b/onionr/static-data/www/private/index.html
index 9a548ebd..a1650883 100755
--- a/onionr/static-data/www/private/index.html
+++ b/onionr/static-data/www/private/index.html
@@ -26,7 +26,7 @@
             <label>Open Site: <input type='text' id='siteViewer' placeholder='Site Hash'> <button id='openSite' class='btn primaryBtn openSiteBtn'>Open Onionr Site</button></label>
             <br>
             <br><br><a class='idLink' href='/mail/'>Mail</a> - <a class='idLink' href='/friends/'>Friend Manager</a> - <a class='idLink' href='/board/'>Circle</a> - 
-                    <a class='idLink' href='/clandestine/'>Clandestine</a>
+                    <a class='idLink' href='/esoteric/'>Esoteric</a>
             <br><br><hr>
             <details class='configArea'>
                 <summary><b>Edit Configuration</b></summary>

From 4e00bdb348a32caf1be23335ec78e77bc4af56e0 Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Sat, 22 Jun 2019 17:54:41 -0500
Subject: [PATCH 07/29] fixed small cli logging bugs

---
 onionr/core.py                                | 73 ++-----------------
 onionr/coredb/blockmetadb/add.py              | 23 ++++++
 .../__init__.py}                              |  0
 onionr/onionrstorage/removeblock.py           | 20 +++++
 onionr/onionrstorage/setdata.py               | 36 +++++++++
 5 files changed, 85 insertions(+), 67 deletions(-)
 create mode 100644 onionr/coredb/blockmetadb/add.py
 rename onionr/{onionrstorage.py => onionrstorage/__init__.py} (100%)
 create mode 100644 onionr/onionrstorage/removeblock.py
 create mode 100644 onionr/onionrstorage/setdata.py

diff --git a/onionr/core.py b/onionr/core.py
index 735358e5..7fd4b3ce 100755
--- a/onionr/core.py
+++ b/onionr/core.py
@@ -25,16 +25,10 @@ import deadsimplekv as simplekv
 import onionrutils, onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions
 import onionrblacklist
 from onionrusers import onionrusers
+from onionrstorage import removeblock, setdata
 import dbcreator, onionrstorage, serializeddata, subprocesspow
 from etc import onionrvalues, powchoice
 
-if sys.version_info < (3, 6):
-    try:
-        import sha3
-    except ModuleNotFoundError:
-        logger.fatal('On Python 3 versions prior to 3.6.x, you need the sha3 module')
-        sys.exit(1)
-
 class Core:
     def __init__(self, torPort=0):
         '''
@@ -149,18 +143,7 @@ class Core:
 
             **You may want blacklist.addToDB(blockHash)
         '''
-
-        if self._utils.validateHash(block):
-            conn = sqlite3.connect(self.blockDB, timeout=30)
-            c = conn.cursor()
-            t = (block,)
-            c.execute('Delete from hashes where hash=?;', t)
-            conn.commit()
-            conn.close()
-            dataSize = sys.getsizeof(onionrstorage.getData(self, block))
-            self._utils.storageCounter.removeBytes(dataSize)
-        else:
-            raise onionrexceptions.InvalidHexHash
+        removeblock.remove_block(self, block)
 
     def createAddressDB(self):
         '''
@@ -186,57 +169,13 @@ class Core:
 
             Should be in hex format!
         '''
-
-        if not os.path.exists(self.blockDB):
-            raise Exception('Block db does not exist')
-        if self._utils.hasBlock(newHash):
-            return
-        conn = sqlite3.connect(self.blockDB, timeout=30)
-        c = conn.cursor()
-        currentTime = self._utils.getEpoch() + self._crypto.secrets.randbelow(301)
-        if selfInsert or dataSaved:
-            selfInsert = 1
-        else:
-            selfInsert = 0
-        data = (newHash, currentTime, '', selfInsert)
-        c.execute('INSERT INTO hashes (hash, dateReceived, dataType, dataSaved) VALUES(?, ?, ?, ?);', data)
-        conn.commit()
-        conn.close()
+        coredb.blockmetadb.add(self, newHash, selfInsert, dataSaved)
 
     def setData(self, data):
         '''
             Set the data assciated with a hash
         '''
-
-        data = data
-        dataSize = sys.getsizeof(data)
-
-        if not type(data) is bytes:
-            data = data.encode()
-
-        dataHash = self._crypto.sha3Hash(data)
-
-        if type(dataHash) is bytes:
-            dataHash = dataHash.decode()
-        blockFileName = self.blockDataLocation + dataHash + '.dat'
-        try:
-            onionrstorage.getData(self, dataHash)
-        except onionrexceptions.NoDataAvailable:
-            if self._utils.storageCounter.addBytes(dataSize) != False:
-                onionrstorage.store(self, data, blockHash=dataHash)
-                conn = sqlite3.connect(self.blockDB, timeout=30)
-                c = conn.cursor()
-                c.execute("UPDATE hashes SET dataSaved=1 WHERE hash = ?;", (dataHash,))
-                conn.commit()
-                conn.close()
-                with open(self.dataNonceFile, 'a') as nonceFile:
-                    nonceFile.write(dataHash + '\n')
-            else:
-                raise onionrexceptions.DiskAllocationReached
-        else:
-            raise Exception("Data is already set for " + dataHash)
-
-        return dataHash
+        return onionrstorage.setdata.set_data(self, data)
 
     def getData(self, hash):
         '''
@@ -513,6 +452,6 @@ class Core:
         '''
         if self._utils.localCommand('/ping', maxWait=10) == 'pong!':
             self.daemonQueueAdd('announceNode')
-            logger.info('Introduction command will be processed.')
+            logger.info('Introduction command will be processed.', terminal=True)
         else:
-            logger.warn('No running node detected. Cannot introduce.')
\ No newline at end of file
+            logger.warn('No running node detected. Cannot introduce.', terminal=True)
\ No newline at end of file
diff --git a/onionr/coredb/blockmetadb/add.py b/onionr/coredb/blockmetadb/add.py
new file mode 100644
index 00000000..d307d351
--- /dev/null
+++ b/onionr/coredb/blockmetadb/add.py
@@ -0,0 +1,23 @@
+import os, sqlite3
+def add_to_block_DB(core_inst, newHash, selfInsert=False, dataSaved=False):
+    '''
+        Add a hash value to the block db
+
+        Should be in hex format!
+    '''
+
+    if not os.path.exists(core_inst.blockDB):
+        raise Exception('Block db does not exist')
+    if core_inst._utils.hasBlock(newHash):
+        return
+    conn = sqlite3.connect(core_inst.blockDB, timeout=30)
+    c = conn.cursor()
+    currentTime = core_inst._utils.getEpoch() + core_inst._crypto.secrets.randbelow(301)
+    if selfInsert or dataSaved:
+        selfInsert = 1
+    else:
+        selfInsert = 0
+    data = (newHash, currentTime, '', selfInsert)
+    c.execute('INSERT INTO hashes (hash, dateReceived, dataType, dataSaved) VALUES(?, ?, ?, ?);', data)
+    conn.commit()
+    conn.close()
\ No newline at end of file
diff --git a/onionr/onionrstorage.py b/onionr/onionrstorage/__init__.py
similarity index 100%
rename from onionr/onionrstorage.py
rename to onionr/onionrstorage/__init__.py
diff --git a/onionr/onionrstorage/removeblock.py b/onionr/onionrstorage/removeblock.py
new file mode 100644
index 00000000..23e574b6
--- /dev/null
+++ b/onionr/onionrstorage/removeblock.py
@@ -0,0 +1,20 @@
+import sys, sqlite3
+import onionrexceptions, onionrstorage
+def remove_block(core_inst, block):
+    '''
+        remove a block from this node (does not automatically blacklist)
+
+        **You may want blacklist.addToDB(blockHash)
+    '''
+
+    if core_inst._utils.validateHash(block):
+        conn = sqlite3.connect(core_inst.blockDB, timeout=30)
+        c = conn.cursor()
+        t = (block,)
+        c.execute('Delete from hashes where hash=?;', t)
+        conn.commit()
+        conn.close()
+        dataSize = sys.getsizeof(onionrstorage.getData(core_inst, block))
+        core_inst._utils.storageCounter.removeBytes(dataSize)
+    else:
+        raise onionrexceptions.InvalidHexHash
\ No newline at end of file
diff --git a/onionr/onionrstorage/setdata.py b/onionr/onionrstorage/setdata.py
new file mode 100644
index 00000000..60378112
--- /dev/null
+++ b/onionr/onionrstorage/setdata.py
@@ -0,0 +1,36 @@
+import sys, sqlite3
+import onionrstorage, onionrexceptions
+def set_data(core_inst, data):
+    '''
+        Set the data assciated with a hash
+    '''
+
+    data = data
+    dataSize = sys.getsizeof(data)
+
+    if not type(data) is bytes:
+        data = data.encode()
+
+    dataHash = core_inst._crypto.sha3Hash(data)
+
+    if type(dataHash) is bytes:
+        dataHash = dataHash.decode()
+    blockFileName = core_inst.blockDataLocation + dataHash + '.dat'
+    try:
+        onionrstorage.getData(core_inst, dataHash)
+    except onionrexceptions.NoDataAvailable:
+        if core_inst._utils.storageCounter.addBytes(dataSize) != False:
+            onionrstorage.store(core_inst, data, blockHash=dataHash)
+            conn = sqlite3.connect(core_inst.blockDB, timeout=30)
+            c = conn.cursor()
+            c.execute("UPDATE hashes SET dataSaved=1 WHERE hash = ?;", (dataHash,))
+            conn.commit()
+            conn.close()
+            with open(core_inst.dataNonceFile, 'a') as nonceFile:
+                nonceFile.write(dataHash + '\n')
+        else:
+            raise onionrexceptions.DiskAllocationReached
+    else:
+        raise Exception("Data is already set for " + dataHash)
+
+    return dataHash
\ No newline at end of file

From 2b64cc9ba4ba71b5533a3209d5e3db413ea05b97 Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Sat, 22 Jun 2019 18:01:55 -0500
Subject: [PATCH 08/29] correct add function name

---
 onionr/core.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/onionr/core.py b/onionr/core.py
index 7fd4b3ce..1a0a465d 100755
--- a/onionr/core.py
+++ b/onionr/core.py
@@ -169,7 +169,7 @@ class Core:
 
             Should be in hex format!
         '''
-        coredb.blockmetadb.add(self, newHash, selfInsert, dataSaved)
+        coredb.blockmetadb.add.add_to_block_DB(self, newHash, selfInsert, dataSaved)
 
     def setData(self, data):
         '''

From e4ec850b60ff33a13716e816f5ad3e612c5b59cd Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Sat, 22 Jun 2019 18:06:16 -0500
Subject: [PATCH 09/29] correctly import add in blockmetadb

---
 onionr/core.py                        | 2 +-
 onionr/coredb/blockmetadb/__init__.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/onionr/core.py b/onionr/core.py
index 1a0a465d..5ab70e7d 100755
--- a/onionr/core.py
+++ b/onionr/core.py
@@ -17,7 +17,7 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import sqlite3, os, sys, time, json, uuid
+import os, sys, time, json, uuid
 import logger, netcontroller, config
 from onionrblockapi import Block
 import coredb
diff --git a/onionr/coredb/blockmetadb/__init__.py b/onionr/coredb/blockmetadb/__init__.py
index fd408d35..9aa991a5 100644
--- a/onionr/coredb/blockmetadb/__init__.py
+++ b/onionr/coredb/blockmetadb/__init__.py
@@ -1,5 +1,5 @@
 import sqlite3
-from . import expiredblocks, updateblockinfo
+from . import expiredblocks, updateblockinfo, add
 def get_block_list(core_inst, dateRec = None, unsaved = False):
     '''
         Get list of our blocks

From 7830484760ddaf78c2d9d5bcc80a3ca5578a7c47 Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Sun, 23 Jun 2019 02:00:27 -0500
Subject: [PATCH 10/29] * started refactoring onionrusers into a full module
 instead of a class * now use daemon threads to prevent process hanging

---
 .../onionrcommunicatortimers.py               |   2 +-
 onionr/core.py                                |   2 +-
 onionr/onionrcommands/daemonlaunch.py         |   8 +-
 .../__init__.py}                              | 177 +-----------------
 onionr/onionrutils/blockmetadata.py           |  68 +++++++
 onionr/onionrutils/localcommand.py            |  31 +++
 onionr/onionrutils/validatemetadata.py        |  77 ++++++++
 .../static-data/default-plugins/flow/main.py  |   2 -
 8 files changed, 188 insertions(+), 179 deletions(-)
 rename onionr/{onionrutils.py => onionrutils/__init__.py} (65%)
 create mode 100644 onionr/onionrutils/blockmetadata.py
 create mode 100644 onionr/onionrutils/localcommand.py
 create mode 100644 onionr/onionrutils/validatemetadata.py

diff --git a/onionr/communicatorutils/onionrcommunicatortimers.py b/onionr/communicatorutils/onionrcommunicatortimers.py
index b7255da4..e765fd66 100755
--- a/onionr/communicatorutils/onionrcommunicatortimers.py
+++ b/onionr/communicatorutils/onionrcommunicatortimers.py
@@ -55,7 +55,7 @@ class OnionrCommunicatorTimers:
                             logger.debug('%s is currently using the maximum number of threads, not starting another.' % self.timerFunction.__name__)
                         else:
                             self.daemonInstance.threadCounts[self.timerFunction.__name__] += 1
-                            newThread = threading.Thread(target=self.timerFunction, args=self.args)
+                            newThread = threading.Thread(target=self.timerFunction, args=self.args, daemon=True)
                             newThread.start()
                 else:
                     self.timerFunction()
diff --git a/onionr/core.py b/onionr/core.py
index 5ab70e7d..ebc35e51 100755
--- a/onionr/core.py
+++ b/onionr/core.py
@@ -201,7 +201,7 @@ class Core:
         '''
             Get a response sent by communicator to the API, by requesting to the API
         '''
-        return coredb.daemonqueue.daemon_queue_get_response(responseID)
+        return coredb.daemonqueue.daemon_queue_get_response(self, responseID)
 
     def clearDaemonQueue(self):
         '''
diff --git a/onionr/onionrcommands/daemonlaunch.py b/onionr/onionrcommands/daemonlaunch.py
index d65a33d6..1aaf7230 100755
--- a/onionr/onionrcommands/daemonlaunch.py
+++ b/onionr/onionrcommands/daemonlaunch.py
@@ -38,8 +38,8 @@ def daemon(o_inst):
         logger.debug('Runcheck file found on daemon start, deleting in advance.')
         os.remove('%s/.runcheck' % (o_inst.onionrCore.dataDir,))
 
-    Thread(target=api.API, args=(o_inst, o_inst.debug, onionr.API_VERSION)).start()
-    Thread(target=api.PublicAPI, args=[o_inst.getClientApi()]).start()
+    Thread(target=api.API, args=(o_inst, o_inst.debug, onionr.API_VERSION), daemon=True).start()
+    Thread(target=api.PublicAPI, args=[o_inst.getClientApi()], daemon=True).start()
 
     apiHost = ''
     while apiHost == '':
@@ -77,7 +77,7 @@ def daemon(o_inst):
         _proper_shutdown(o_inst)
 
     o_inst.onionrCore.torPort = net.socksPort
-    communicatorThread = Thread(target=communicator.startCommunicator, args=(o_inst, str(net.socksPort)))
+    communicatorThread = Thread(target=communicator.startCommunicator, args=(o_inst, str(net.socksPort)), daemon=True)
     communicatorThread.start()
     
     while o_inst.communicatorInst is None:
@@ -106,7 +106,7 @@ def daemon(o_inst):
     o_inst.onionrUtils.localCommand('shutdown')
 
     net.killTor()
-    time.sleep(3)
+    time.sleep(8) # Time to allow threads to finish, if not any "daemon" threads will be slaughtered http://docs.python.org/library/threading.html#threading.Thread.daemon
     o_inst.deleteRunFiles()
     return
 
diff --git a/onionr/onionrutils.py b/onionr/onionrutils/__init__.py
similarity index 65%
rename from onionr/onionrutils.py
rename to onionr/onionrutils/__init__.py
index 4aa02d76..67d7f8f0 100755
--- a/onionr/onionrutils.py
+++ b/onionr/onionrutils/__init__.py
@@ -29,12 +29,8 @@ import onionrevents
 import storagecounter
 from etc import pgpwords, onionrvalues
 from onionrusers import onionrusers 
-if sys.version_info < (3, 6):
-    try:
-        import sha3
-    except ModuleNotFoundError:
-        logger.fatal('On Python 3 versions prior to 3.6.x, you need the sha3 module')
-        sys.exit(1)
+from . import localcommand, blockmetadata, validatemetadata
+
 config.reload()
 class OnionrUtils:
     '''
@@ -44,25 +40,11 @@ class OnionrUtils:
         #self.fingerprintFile = 'data/own-fingerprint.txt' #TODO Remove since probably not needed
         self._core = coreInstance # onionr core instance
 
-        self.timingToken = '' # for when we make local connections to our http api, to bypass timing attack defense mechanism
         self.avoidDupe = [] # list used to prevent duplicate requests per peer for certain actions
         self.peerProcessing = {} # dict of current peer actions: peer, actionList
         self.storageCounter = storagecounter.StorageCounter(self._core) # used to keep track of how much data onionr is using on disk
         return
 
-    def getTimeBypassToken(self):
-        '''
-            Load our timingToken from disk for faster local HTTP API
-        '''
-        try:
-            if os.path.exists(self._core.dataDir + 'time-bypass.txt'):
-                with open(self._core.dataDir + 'time-bypass.txt', 'r') as bypass:
-                    self.timingToken = bypass.read()
-        except Exception as error:
-            logger.error('Failed to fetch time bypass token.', error = error)
-
-        return self.timingToken
-
     def getRoundedEpoch(self, roundS=60):
         '''
             Returns the epoch, rounded down to given seconds (Default 60)
@@ -85,32 +67,7 @@ class OnionrUtils:
         '''
             Send a command to the local http API server, securely. Intended for local clients, DO NOT USE for remote peers.
         '''
-        self.getTimeBypassToken()
-        # TODO: URL encode parameters, just as an extra measure. May not be needed, but should be added regardless.
-        hostname = ''
-        waited = 0
-        while hostname == '':
-            try:
-                hostname = self.getClientAPIServer()
-            except FileNotFoundError:
-                time.sleep(1)
-                waited += 1
-                if waited == maxWait:
-                    return False
-        if data != '':
-            data = '&data=' + urllib.parse.quote_plus(data)
-        payload = 'http://%s/%s%s' % (hostname, command, data)
-        try:
-            if post:
-                retData = requests.post(payload, data=postData, headers={'token': config.get('client.webpassword'), 'Connection':'close'}, timeout=(maxWait, maxWait)).text
-            else:
-                retData = requests.get(payload, headers={'token': config.get('client.webpassword'), 'Connection':'close'}, timeout=(maxWait, maxWait)).text
-        except Exception as error:
-            if not silent:
-                logger.error('Failed to make local request (command: %s):%s' % (command, error))
-            retData = False
-
-        return retData
+        return localcommand.local_command(self, command, data, silent, post, postData, maxWait)
 
     def getHumanReadableID(self, pub=''):
         '''gets a human readable ID from a public key'''
@@ -130,64 +87,13 @@ class OnionrUtils:
             metadata, meta (meta being internal metadata, which will be 
             returned as an encrypted base64 string if it is encrypted, dict if not).
         '''
-        meta = {}
-        metadata = {}
-        data = blockData
-        try:
-            blockData = blockData.encode()
-        except AttributeError:
-            pass
-
-        try:
-            metadata = json.loads(blockData[:blockData.find(b'\n')].decode())
-        except json.decoder.JSONDecodeError:
-            pass
-        else:
-            data = blockData[blockData.find(b'\n'):].decode()
-
-            if not metadata['encryptType'] in ('asym', 'sym'):
-                try:
-                    meta = json.loads(metadata['meta'])
-                except KeyError:
-                    pass
-            meta = metadata['meta']
-        return (metadata, meta, data)
+        return blockmetadata.get_block_metadata_from_data(self, blockData)
 
     def processBlockMetadata(self, blockHash):
         '''
             Read metadata from a block and cache it to the block database
         '''
-        curTime = self.getRoundedEpoch(roundS=60)
-        myBlock = Block(blockHash, self._core)
-        if myBlock.isEncrypted:
-            myBlock.decrypt()
-        if (myBlock.isEncrypted and myBlock.decrypted) or (not myBlock.isEncrypted):
-            blockType = myBlock.getMetadata('type') # we would use myBlock.getType() here, but it is bugged with encrypted blocks
-            signer = self.bytesToStr(myBlock.signer)
-            valid = myBlock.verifySig()
-            if myBlock.getMetadata('newFSKey') is not None:
-                onionrusers.OnionrUser(self._core, signer).addForwardKey(myBlock.getMetadata('newFSKey'))
-                
-            try:
-                if len(blockType) <= 10:
-                    self._core.updateBlockInfo(blockHash, 'dataType', blockType)
-            except TypeError:
-                logger.warn("Missing block information")
-                pass
-            # Set block expire time if specified
-            try:
-                expireTime = myBlock.getHeader('expire')
-                assert len(str(int(expireTime))) < 20 # test that expire time is an integer of sane length (for epoch)
-            except (AssertionError, ValueError, TypeError) as e:
-                expireTime = onionrvalues.OnionrValues().default_expire + curTime
-            finally:
-                self._core.updateBlockInfo(blockHash, 'expire', expireTime)
-            if not blockType is None:
-                self._core.updateBlockInfo(blockHash, 'dataType', blockType)
-            onionrevents.event('processblocks', data = {'block': myBlock, 'type': blockType, 'signer': signer, 'validSig': valid}, onionr = self._core.onionrInst)
-        else:
-            pass
-            #logger.debug('Not processing metadata on encrypted block we cannot decrypt.')
+        return blockmetadata.process_block_metadata(self, blockHash)
 
     def escapeAnsi(self, line):
         '''
@@ -243,78 +149,7 @@ class OnionrUtils:
 
     def validateMetadata(self, metadata, blockData):
         '''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string'''
-        # TODO, make this check sane sizes
-        retData = False
-        maxClockDifference = 120
-
-        # convert to dict if it is json string
-        if type(metadata) is str:
-            try:
-                metadata = json.loads(metadata)
-            except json.JSONDecodeError:
-                pass
-
-        # Validate metadata dict for invalid keys to sizes that are too large
-        maxAge = config.get("general.max_block_age", onionrvalues.OnionrValues().default_expire)
-        if type(metadata) is dict:
-            for i in metadata:
-                try:
-                    self._core.requirements.blockMetadataLengths[i]
-                except KeyError:
-                    logger.warn('Block has invalid metadata key ' + i)
-                    break
-                else:
-                    testData = metadata[i]
-                    try:
-                        testData = len(testData)
-                    except (TypeError, AttributeError) as e:
-                        testData = len(str(testData))
-                    if self._core.requirements.blockMetadataLengths[i] < testData:
-                        logger.warn('Block metadata key ' + i + ' exceeded maximum size')
-                        break
-                if i == 'time':
-                    if not self.isIntegerString(metadata[i]):
-                        logger.warn('Block metadata time stamp is not integer string or int')
-                        break
-                    isFuture = (metadata[i] - self.getEpoch())
-                    if isFuture > maxClockDifference:
-                        logger.warn('Block timestamp is skewed to the future over the max %s: %s' (maxClockDifference, isFuture))
-                        break
-                    if (self.getEpoch() - metadata[i]) > maxAge:
-                        logger.warn('Block is outdated: %s' % (metadata[i],))
-                        break
-                elif i == 'expire':
-                    try:
-                        assert int(metadata[i]) > self.getEpoch()
-                    except AssertionError:
-                        logger.warn('Block is expired: %s less than %s' % (metadata[i], self.getEpoch()))
-                        break
-                elif i == 'encryptType':
-                    try:
-                        assert metadata[i] in ('asym', 'sym', '')
-                    except AssertionError:
-                        logger.warn('Invalid encryption mode')
-                        break
-            else:
-                # if metadata loop gets no errors, it does not break, therefore metadata is valid
-                # make sure we do not have another block with the same data content (prevent data duplication and replay attacks)
-                nonce = self._core._utils.bytesToStr(self._core._crypto.sha3Hash(blockData))
-                try:
-                    with open(self._core.dataNonceFile, 'r') as nonceFile:
-                        if nonce in nonceFile.read():
-                            retData = False # we've seen that nonce before, so we can't pass metadata
-                            raise onionrexceptions.DataExists
-                except FileNotFoundError:
-                    retData = True
-                except onionrexceptions.DataExists:
-                    # do not set retData to True, because nonce has been seen before
-                    pass
-                else:
-                    retData = True
-        else:
-            logger.warn('In call to utils.validateMetadata, metadata must be JSON string or a dictionary object')
-
-        return retData
+        return validatemetadata.validate_metadata(self, metadata, blockData)
 
     def validatePubKey(self, key):
         '''
diff --git a/onionr/onionrutils/blockmetadata.py b/onionr/onionrutils/blockmetadata.py
new file mode 100644
index 00000000..0515ffc7
--- /dev/null
+++ b/onionr/onionrutils/blockmetadata.py
@@ -0,0 +1,68 @@
+import json
+import logger, onionrevents
+from onionrusers import onionrusers
+from etc import onionrvalues
+from onionrblockapi import Block
+def get_block_metadata_from_data(utils_inst, blockData):
+    '''
+        accepts block contents as string, returns a tuple of 
+        metadata, meta (meta being internal metadata, which will be 
+        returned as an encrypted base64 string if it is encrypted, dict if not).
+    '''
+    meta = {}
+    metadata = {}
+    data = blockData
+    try:
+        blockData = blockData.encode()
+    except AttributeError:
+        pass
+
+    try:
+        metadata = json.loads(blockData[:blockData.find(b'\n')].decode())
+    except json.decoder.JSONDecodeError:
+        pass
+    else:
+        data = blockData[blockData.find(b'\n'):].decode()
+
+        if not metadata['encryptType'] in ('asym', 'sym'):
+            try:
+                meta = json.loads(metadata['meta'])
+            except KeyError:
+                pass
+        meta = metadata['meta']
+    return (metadata, meta, data)
+
+def process_block_metadata(utils_inst, blockHash):
+    '''
+        Read metadata from a block and cache it to the block database
+    '''
+    curTime = utils_inst.getRoundedEpoch(roundS=60)
+    myBlock = Block(blockHash, utils_inst._core)
+    if myBlock.isEncrypted:
+        myBlock.decrypt()
+    if (myBlock.isEncrypted and myBlock.decrypted) or (not myBlock.isEncrypted):
+        blockType = myBlock.getMetadata('type') # we would use myBlock.getType() here, but it is bugged with encrypted blocks
+        signer = utils_inst.bytesToStr(myBlock.signer)
+        valid = myBlock.verifySig()
+        if myBlock.getMetadata('newFSKey') is not None:
+            onionrusers.OnionrUser(utils_inst._core, signer).addForwardKey(myBlock.getMetadata('newFSKey'))
+            
+        try:
+            if len(blockType) <= 10:
+                utils_inst._core.updateBlockInfo(blockHash, 'dataType', blockType)
+        except TypeError:
+            logger.warn("Missing block information")
+            pass
+        # Set block expire time if specified
+        try:
+            expireTime = myBlock.getHeader('expire')
+            assert len(str(int(expireTime))) < 20 # test that expire time is an integer of sane length (for epoch)
+        except (AssertionError, ValueError, TypeError) as e:
+            expireTime = onionrvalues.OnionrValues().default_expire + curTime
+        finally:
+            utils_inst._core.updateBlockInfo(blockHash, 'expire', expireTime)
+        if not blockType is None:
+            utils_inst._core.updateBlockInfo(blockHash, 'dataType', blockType)
+        onionrevents.event('processblocks', data = {'block': myBlock, 'type': blockType, 'signer': signer, 'validSig': valid}, onionr = utils_inst._core.onionrInst)
+    else:
+        pass
\ No newline at end of file
diff --git a/onionr/onionrutils/localcommand.py b/onionr/onionrutils/localcommand.py
new file mode 100644
index 00000000..fff050c8
--- /dev/null
+++ b/onionr/onionrutils/localcommand.py
@@ -0,0 +1,31 @@
+import urllib, requests, time
+import logger
+def local_command(utils_inst, command, data='', silent = True, post=False, postData = {}, maxWait=20):
+    '''
+        Send a command to the local http API server, securely. Intended for local clients, DO NOT USE for remote peers.
+    '''
+    # TODO: URL encode parameters, just as an extra measure. May not be needed, but should be added regardless.
+    hostname = ''
+    waited = 0
+    while hostname == '':
+        try:
+            hostname = utils_inst.getClientAPIServer()
+        except FileNotFoundError:
+            time.sleep(1)
+            waited += 1
+            if waited == maxWait:
+                return False
+    if data != '':
+        data = '&data=' + urllib.parse.quote_plus(data)
+    payload = 'http://%s/%s%s' % (hostname, command, data)
+    try:
+        if post:
+            retData = requests.post(payload, data=postData, headers={'token': config.get('client.webpassword'), 'Connection':'close'}, timeout=(maxWait, maxWait)).text
+        else:
+            retData = requests.get(payload, headers={'token': config.get('client.webpassword'), 'Connection':'close'}, timeout=(maxWait, maxWait)).text
+    except Exception as error:
+        if not silent:
+            logger.error('Failed to make local request (command: %s):%s' % (command, error), terminal=True)
+        retData = False
+
+    return retData
\ No newline at end of file
diff --git a/onionr/onionrutils/validatemetadata.py b/onionr/onionrutils/validatemetadata.py
new file mode 100644
index 00000000..c4bcc4a8
--- /dev/null
+++ b/onionr/onionrutils/validatemetadata.py
@@ -0,0 +1,77 @@
+import json
+import logger, onionrexceptions
+from etc import onionrvalues
+def validate_metadata(utils_inst, metadata, blockData):
+    '''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string'''
+    # TODO, make this check sane sizes
+    retData = False
+    maxClockDifference = 120
+
+    # convert to dict if it is json string
+    if type(metadata) is str:
+        try:
+            metadata = json.loads(metadata)
+        except json.JSONDecodeError:
+            pass
+
+    # Validate metadata dict for invalid keys to sizes that are too large
+    maxAge = utils_inst._coreconfig.get("general.max_block_age", onionrvalues.OnionrValues().default_expire)
+    if type(metadata) is dict:
+        for i in metadata:
+            try:
+                utils_inst._core.requirements.blockMetadataLengths[i]
+            except KeyError:
+                logger.warn('Block has invalid metadata key ' + i)
+                break
+            else:
+                testData = metadata[i]
+                try:
+                    testData = len(testData)
+                except (TypeError, AttributeError) as e:
+                    testData = len(str(testData))
+                if utils_inst._core.requirements.blockMetadataLengths[i] < testData:
+                    logger.warn('Block metadata key ' + i + ' exceeded maximum size')
+                    break
+            if i == 'time':
+                if not utils_inst.isIntegerString(metadata[i]):
+                    logger.warn('Block metadata time stamp is not integer string or int')
+                    break
+                isFuture = (metadata[i] - utils_inst.getEpoch())
+                if isFuture > maxClockDifference:
+                    logger.warn('Block timestamp is skewed to the future over the max %s: %s' (maxClockDifference, isFuture))
+                    break
+                if (utils_inst.getEpoch() - metadata[i]) > maxAge:
+                    logger.warn('Block is outdated: %s' % (metadata[i],))
+                    break
+            elif i == 'expire':
+                try:
+                    assert int(metadata[i]) > utils_inst.getEpoch()
+                except AssertionError:
+                    logger.warn('Block is expired: %s less than %s' % (metadata[i], utils_inst.getEpoch()))
+                    break
+            elif i == 'encryptType':
+                try:
+                    assert metadata[i] in ('asym', 'sym', '')
+                except AssertionError:
+                    logger.warn('Invalid encryption mode')
+                    break
+        else:
+            # if metadata loop gets no errors, it does not break, therefore metadata is valid
+            # make sure we do not have another block with the same data content (prevent data duplication and replay attacks)
+            nonce = utils_inst._core._utils.bytesToStr(utils_inst._core._crypto.sha3Hash(blockData))
+            try:
+                with open(utils_inst._core.dataNonceFile, 'r') as nonceFile:
+                    if nonce in nonceFile.read():
+                        retData = False # we've seen that nonce before, so we can't pass metadata
+                        raise onionrexceptions.DataExists
+            except FileNotFoundError:
+                retData = True
+            except onionrexceptions.DataExists:
+                # do not set retData to True, because nonce has been seen before
+                pass
+            else:
+                retData = True
+    else:
+        logger.warn('In call to utils.validateMetadata, metadata must be JSON string or a dictionary object')
+
+    return retData
\ No newline at end of file
diff --git a/onionr/static-data/default-plugins/flow/main.py b/onionr/static-data/default-plugins/flow/main.py
index d1c2c87d..5f61aa38 100755
--- a/onionr/static-data/default-plugins/flow/main.py
+++ b/onionr/static-data/default-plugins/flow/main.py
@@ -75,10 +75,8 @@ class OnionrFlow:
                 for block in self.myCore.getBlocksByType('txt'):
                     block = Block(block)
                     if block.getMetadata('ch') != self.channel:
-                        #print('not chan', block.getMetadata('ch'))
                         continue
                     if block.getHash() in self.alreadyOutputed:
-                        #print('already')
                         continue
                     if not self.flowRunning:
                         break

From 0d258e1a16b42d746a4d86594456ed57cd3f16ab Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Sun, 23 Jun 2019 02:25:40 -0500
Subject: [PATCH 11/29] * more utils refactoring, requests this time

---
 onionr/onionrutils/__init__.py      | 81 ++---------------------------
 onionr/onionrutils/basicrequests.py | 71 +++++++++++++++++++++++++
 run-windows-dev.bat                 |  3 +-
 3 files changed, 76 insertions(+), 79 deletions(-)
 create mode 100644 onionr/onionrutils/basicrequests.py

diff --git a/onionr/onionrutils/__init__.py b/onionr/onionrutils/__init__.py
index 67d7f8f0..a7efbae7 100755
--- a/onionr/onionrutils/__init__.py
+++ b/onionr/onionrutils/__init__.py
@@ -24,12 +24,11 @@ import nacl.signing, nacl.encoding
 import unpaddedbase32
 from onionrblockapi import Block
 import onionrexceptions, config, logger
-from onionr import API_VERSION
 import onionrevents
 import storagecounter
 from etc import pgpwords, onionrvalues
 from onionrusers import onionrusers 
-from . import localcommand, blockmetadata, validatemetadata
+from . import localcommand, blockmetadata, validatemetadata, basicrequests
 
 config.reload()
 class OnionrUtils:
@@ -279,23 +278,6 @@ class OnionrUtils:
         if not exist:
             logger.info('No blocks found to import')
 
-    def progressBar(self, value = 0, endvalue = 100, width = None):
-        '''
-            Outputs a progress bar with a percentage. Write \n after use.
-        '''
-
-        if width is None or height is None:
-            width, height = shutil.get_terminal_size((80, 24))
-
-        bar_length = width - 6
-
-        percent = float(value) / endvalue
-        arrow = '─' * int(round(percent * bar_length)-1) + '>'
-        spaces = ' ' * (bar_length - len(arrow))
-
-        sys.stdout.write("\r┣{0}┫ {1}%".format(arrow + spaces, int(round(percent * 100))))
-        sys.stdout.flush()
-
     def getEpoch(self):
         '''returns epoch'''
         return math.floor(time.time())
@@ -304,70 +286,13 @@ class OnionrUtils:
         '''
         Do a POST request through a local tor or i2p instance
         '''
-        if proxyType == 'tor':
-            if port == 0:
-                port = self._core.torPort
-            proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
-        elif proxyType == 'i2p':
-            proxies = {'http': 'http://127.0.0.1:4444'}
-        else:
-            return
-        headers = {'user-agent': 'PyOnionr', 'Connection':'close'}
-        try:
-            proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
-            r = requests.post(url, data=data, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30))
-            retData = r.text
-        except KeyboardInterrupt:
-            raise KeyboardInterrupt
-        except requests.exceptions.RequestException as e:
-            logger.debug('Error: %s' % str(e))
-            retData = False
-        return retData
+        return basicrequests.do_post_request(self, url, data, port, proxyType)
 
     def doGetRequest(self, url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=False):
         '''
         Do a get request through a local tor or i2p instance
         '''
-        retData = False
-        if proxyType == 'tor':
-            if port == 0:
-                raise onionrexceptions.MissingPort('Socks port required for Tor HTTP get request')
-            proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
-        elif proxyType == 'i2p':
-            proxies = {'http': 'http://127.0.0.1:4444'}
-        else:
-            return
-        headers = {'user-agent': 'PyOnionr', 'Connection':'close'}
-        response_headers = dict()
-        try:
-            proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
-            r = requests.get(url, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30), )
-            # Check server is using same API version as us
-            if not ignoreAPI:
-                try:
-                    response_headers = r.headers
-                    if r.headers['X-API'] != str(API_VERSION):
-                        raise onionrexceptions.InvalidAPIVersion
-                except KeyError:
-                    raise onionrexceptions.InvalidAPIVersion
-            retData = r.text
-        except KeyboardInterrupt:
-            raise KeyboardInterrupt
-        except ValueError as e:
-            pass
-        except onionrexceptions.InvalidAPIVersion:
-            if 'X-API' in response_headers:
-                logger.debug('Using API version %s. Cannot communicate with node\'s API version of %s.' % (API_VERSION, response_headers['X-API']))
-            else:
-                logger.debug('Using API version %s. API version was not sent with the request.' % API_VERSION)
-        except requests.exceptions.RequestException as e:
-            if not 'ConnectTimeoutError' in str(e) and not 'Request rejected or failed' in str(e):
-                logger.debug('Error: %s' % str(e))
-            retData = False
-        if returnHeaders:
-            return (retData, response_headers)
-        else:
-            return retData
+        return basicrequests.do_get_request(self, url, port, proxyType, ignoreAPI, returnHeaders)
 
     @staticmethod
     def strToBytes(data):
diff --git a/onionr/onionrutils/basicrequests.py b/onionr/onionrutils/basicrequests.py
new file mode 100644
index 00000000..f9432928
--- /dev/null
+++ b/onionr/onionrutils/basicrequests.py
@@ -0,0 +1,71 @@
+import requests
+import logger, onionrexceptions
+from onionr import API_VERSION
+def do_post_request(utils_inst, url, data={}, port=0, proxyType='tor'):
+    '''
+    Do a POST request through a local tor or i2p instance
+    '''
+    if proxyType == 'tor':
+        if port == 0:
+            port = utils_inst._core.torPort
+        proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
+    elif proxyType == 'i2p':
+        proxies = {'http': 'http://127.0.0.1:4444'}
+    else:
+        return
+    headers = {'user-agent': 'PyOnionr', 'Connection':'close'}
+    try:
+        proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
+        r = requests.post(url, data=data, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30))
+        retData = r.text
+    except KeyboardInterrupt:
+        raise KeyboardInterrupt
+    except requests.exceptions.RequestException as e:
+        logger.debug('Error: %s' % str(e))
+        retData = False
+    return retData
+
+def do_get_request(utils_inst, url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=False):
+    '''
+    Do a get request through a local tor or i2p instance
+    '''
+    retData = False
+    if proxyType == 'tor':
+        if port == 0:
+            raise onionrexceptions.MissingPort('Socks port required for Tor HTTP get request')
+        proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
+    elif proxyType == 'i2p':
+        proxies = {'http': 'http://127.0.0.1:4444'}
+    else:
+        return
+    headers = {'user-agent': 'PyOnionr', 'Connection':'close'}
+    response_headers = dict()
+    try:
+        proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
+        r = requests.get(url, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30), )
+        # Check server is using same API version as us
+        if not ignoreAPI:
+            try:
+                response_headers = r.headers
+                if r.headers['X-API'] != str(API_VERSION):
+                    raise onionrexceptions.InvalidAPIVersion
+            except KeyError:
+                raise onionrexceptions.InvalidAPIVersion
+        retData = r.text
+    except KeyboardInterrupt:
+        raise KeyboardInterrupt
+    except ValueError as e:
+        pass
+    except onionrexceptions.InvalidAPIVersion:
+        if 'X-API' in response_headers:
+            logger.debug('Using API version %s. Cannot communicate with node\'s API version of %s.' % (API_VERSION, response_headers['X-API']))
+        else:
+            logger.debug('Using API version %s. API version was not sent with the request.' % API_VERSION)
+    except requests.exceptions.RequestException as e:
+        if not 'ConnectTimeoutError' in str(e) and not 'Request rejected or failed' in str(e):
+            logger.debug('Error: %s' % str(e))
+        retData = False
+    if returnHeaders:
+        return (retData, response_headers)
+    else:
+        return retData
\ No newline at end of file
diff --git a/run-windows-dev.bat b/run-windows-dev.bat
index 31ba8c88..50de1152 100755
--- a/run-windows-dev.bat
+++ b/run-windows-dev.bat
@@ -1,6 +1,7 @@
 @echo off
+echo This script is only intended for use in Onionr development, as it uses a random profile.
 set ONIONR_HOME=data%random%
-echo Using %ONIONR_HOME%
+echo Using profile: %ONIONR_HOME%
 setlocal
 chdir onionr
 python onionr.py %*

From d3783400991f1e6199ce3ad2c547317005c8219b Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Sun, 23 Jun 2019 12:41:07 -0500
Subject: [PATCH 12/29] * further splitting onionrutils into a module

---
 onionr/communicator.py                        |   7 +-
 onionr/communicatorutils/connectnewpeers.py   |   3 +-
 .../communicatorutils/daemonqueuehandler.py   |   5 +-
 onionr/communicatorutils/lookupadders.py      |   3 +-
 onionr/communicatorutils/netcheck.py          |   6 +-
 onionr/communicatorutils/servicecreator.py    |   4 +-
 onionr/communicatorutils/uploadblocks.py      |   3 +-
 onionr/core.py                                |   7 +-
 onionr/coredb/daemonqueue/__init__.py         |   4 +-
 onionr/coredb/keydb/addkeys.py                |   8 +-
 onionr/coredb/keydb/removekeys.py             |   4 +-
 onionr/httpapi/miscpublicapi/announce.py      |   4 +-
 onionr/onionrcommands/__init__.py             |   5 +-
 onionr/onionrcommands/daemonlaunch.py         |   9 +-
 onionr/onionrcommands/onionrstatistics.py     |   3 +-
 onionr/onionrcommands/openwebinterface.py     |   3 +-
 onionr/onionrcommands/resettor.py             |   4 +-
 onionr/onionrpluginapi.py                     |   3 +-
 onionr/onionrservices/__init__.py             |   3 +-
 onionr/onionrservices/bootstrapservice.py     |   3 +-
 onionr/onionrutils/__init__.py                | 156 +-----------------
 onionr/onionrutils/checkcommunicator.py       |  19 +++
 onionr/onionrutils/getclientapiserver.py      |  10 ++
 onionr/onionrutils/importnewblocks.py         |  28 ++++
 onionr/onionrutils/localcommand.py            |   5 +-
 onionr/onionrutils/stringvalidators.py        |  97 +++++++++++
 onionr/onionrutils/validatemetadata.py        |   5 +-
 .../static-data/default-plugins/cliui/main.py |   3 +-
 .../default-plugins/esoteric/peerserver.py    |   5 +-
 .../default-plugins/pluginmanager/main.py     |   3 +-
 onionr/tests/test_stringvalidations.py        |  11 +-
 31 files changed, 234 insertions(+), 199 deletions(-)
 create mode 100644 onionr/onionrutils/checkcommunicator.py
 create mode 100644 onionr/onionrutils/getclientapiserver.py
 create mode 100644 onionr/onionrutils/importnewblocks.py
 create mode 100644 onionr/onionrutils/stringvalidators.py

diff --git a/onionr/communicator.py b/onionr/communicator.py
index d59b76da..57917b9a 100755
--- a/onionr/communicator.py
+++ b/onionr/communicator.py
@@ -27,6 +27,7 @@ from communicatorutils import downloadblocks, lookupblocks, lookupadders
 from communicatorutils import servicecreator, connectnewpeers, uploadblocks
 from communicatorutils import daemonqueuehandler, announcenode, deniableinserts
 from communicatorutils import cooldownpeer, housekeeping, netcheck
+from onionrutils import localcommand
 from etc import humanreadabletime
 import onionrservices, onionr, onionrproofs
 
@@ -184,7 +185,7 @@ class OnionrCommunicatorDaemon:
         else:
             for server in self.service_greenlets:
                 server.stop()
-        self._core._utils.localCommand('shutdown') # shutdown the api
+        localcommand.local_command(self._core, 'shutdown') # shutdown the api
         time.sleep(0.5)
 
     def lookupAdders(self):
@@ -364,9 +365,9 @@ class OnionrCommunicatorDaemon:
 
     def detectAPICrash(self):
         '''exit if the api server crashes/stops'''
-        if self._core._utils.localCommand('ping', silent=False) not in ('pong', 'pong!'):
+        if localcommand.local_command(self._core, 'ping', silent=False) not in ('pong', 'pong!'):
             for i in range(300):
-                if self._core._utils.localCommand('ping') in ('pong', 'pong!') or self.shutdown:
+                if localcommand.local_command(self._core, 'ping') in ('pong', 'pong!') or self.shutdown:
                     break # break for loop
                 time.sleep(1)
             else:
diff --git a/onionr/communicatorutils/connectnewpeers.py b/onionr/communicatorutils/connectnewpeers.py
index 96e4ecac..25929c24 100755
--- a/onionr/communicatorutils/connectnewpeers.py
+++ b/onionr/communicatorutils/connectnewpeers.py
@@ -20,6 +20,7 @@
 import time, sys
 import onionrexceptions, logger, onionrpeers
 from utils import networkmerger
+from onionrutils import stringvalidators
 # secrets module was added into standard lib in 3.6+
 if sys.version_info[0] == 3 and sys.version_info[1] < 6:
     from dependencies import secrets
@@ -30,7 +31,7 @@ def connect_new_peer_to_communicator(comm_inst, peer='', useBootstrap=False):
     retData = False
     tried = comm_inst.offlinePeers
     if peer != '':
-        if comm_inst._core._utils.validateID(peer):
+        if stringvalidators.validate_transport(peer):
             peerList = [peer]
         else:
             raise onionrexceptions.InvalidAddress('Will not attempt connection test to invalid address')
diff --git a/onionr/communicatorutils/daemonqueuehandler.py b/onionr/communicatorutils/daemonqueuehandler.py
index ddbb0783..c11cc929 100755
--- a/onionr/communicatorutils/daemonqueuehandler.py
+++ b/onionr/communicatorutils/daemonqueuehandler.py
@@ -19,6 +19,7 @@
 '''
 import logger
 import onionrevents as events
+from onionrutils import localcommand
 def handle_daemon_commands(comm_inst):
     cmd = comm_inst._core.daemonQueue()
     response = ''
@@ -39,7 +40,7 @@ def handle_daemon_commands(comm_inst):
             if response == '':
                 response = 'none'
         elif cmd[0] == 'localCommand':
-            response = comm_inst._core._utils.localCommand(cmd[1])
+            response = localcommand.local_command(comm_inst._core, cmd[1])
         elif cmd[0] == 'pex':
             for i in comm_inst.timers:
                 if i.timerFunction.__name__ == 'lookupAdders':
@@ -49,7 +50,7 @@ def handle_daemon_commands(comm_inst):
 
         if cmd[0] not in ('', None):
             if response != '':
-                comm_inst._core._utils.localCommand('queueResponseAdd/' + cmd[4], post=True, postData={'data': response})
+                localcommand.local_command(comm_inst._core, 'queueResponseAdd/' + cmd[4], post=True, postData={'data': response})
         response = ''
 
     comm_inst.decrementThreadCount('daemonCommands')
\ No newline at end of file
diff --git a/onionr/communicatorutils/lookupadders.py b/onionr/communicatorutils/lookupadders.py
index 4f861682..fc4527dc 100755
--- a/onionr/communicatorutils/lookupadders.py
+++ b/onionr/communicatorutils/lookupadders.py
@@ -18,6 +18,7 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 import logger
+from onionrutils import stringvalidators
 
 def lookup_new_peer_transports_with_communicator(comm_inst):
     logger.info('Looking up new addresses...')
@@ -39,7 +40,7 @@ def lookup_new_peer_transports_with_communicator(comm_inst):
         invalid = []
         for x in newPeers:
             x = x.strip()
-            if not comm_inst._core._utils.validateID(x) or x in comm_inst.newPeers or x == comm_inst._core.hsAddress:
+            if not stringvalidators.validate_transport(x) or x in comm_inst.newPeers or x == comm_inst._core.hsAddress:
                 # avoid adding if its our address
                 invalid.append(x)
         for x in invalid:
diff --git a/onionr/communicatorutils/netcheck.py b/onionr/communicatorutils/netcheck.py
index c5e95906..688feeea 100755
--- a/onionr/communicatorutils/netcheck.py
+++ b/onionr/communicatorutils/netcheck.py
@@ -20,17 +20,19 @@
 '''
 import logger
 from utils import netutils
+from onionrutils import localcommand
 def net_check(comm_inst):
     '''Check if we are connected to the internet or not when we can't connect to any peers'''
     rec = False # for detecting if we have received incoming connections recently
+    c = comm_inst._core
     if len(comm_inst.onlinePeers) == 0:
         try:
-            if (comm_inst._core._utils.getEpoch() - int(comm_inst._core._utils.localCommand('/lastconnect'))) <= 60:
+            if (c._utils.getEpoch() - int(localcommand.local_command(c, '/lastconnect'))) <= 60:
                 comm_inst.isOnline = True
                 rec = True
         except ValueError:
             pass
-        if not rec and not netutils.checkNetwork(comm_inst._core._utils, torPort=comm_inst.proxyPort):
+        if not rec and not netutils.checkNetwork(c._utils, torPort=comm_inst.proxyPort):
             if not comm_inst.shutdown:
                 logger.warn('Network check failed, are you connected to the Internet, and is Tor working?')
             comm_inst.isOnline = False
diff --git a/onionr/communicatorutils/servicecreator.py b/onionr/communicatorutils/servicecreator.py
index c1f2c7e2..94fc51c8 100755
--- a/onionr/communicatorutils/servicecreator.py
+++ b/onionr/communicatorutils/servicecreator.py
@@ -18,6 +18,8 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 import communicator, onionrblockapi
+from onionrutils import stringvalidators
+
 def service_creator(daemon):
     assert isinstance(daemon, communicator.OnionrCommunicatorDaemon)
     core = daemon._core
@@ -30,7 +32,7 @@ def service_creator(daemon):
         if not b in daemon.active_services:
             bl = onionrblockapi.Block(b, core=core, decrypt=True)
             bs = utils.bytesToStr(bl.bcontent) + '.onion'
-            if utils.validatePubKey(bl.signer) and utils.validateID(bs):
+            if utils.validatePubKey(bl.signer) and stringvalidators.validate_transport(bs):
                 signer = utils.bytesToStr(bl.signer)
                 daemon.active_services.append(b)
                 daemon.active_services.append(signer)
diff --git a/onionr/communicatorutils/uploadblocks.py b/onionr/communicatorutils/uploadblocks.py
index 69d0353d..497e07d5 100755
--- a/onionr/communicatorutils/uploadblocks.py
+++ b/onionr/communicatorutils/uploadblocks.py
@@ -20,6 +20,7 @@
 import logger
 from communicatorutils import proxypicker
 import onionrblockapi as block
+from onionrutils import localcommand
 
 def upload_blocks_from_communicator(comm_inst):
     # when inserting a block, we try to upload it to a few peers to add some deniability
@@ -42,7 +43,7 @@ def upload_blocks_from_communicator(comm_inst):
                 proxyType = proxypicker.pick_proxy(peer)
                 logger.info("Uploading block to " + peer)
                 if not comm_inst._core._utils.doPostRequest(url, data=data, proxyType=proxyType) == False:
-                    comm_inst._core._utils.localCommand('waitforshare/' + bl, post=True)
+                    localcommand.local_command(comm_inst._core, 'waitforshare/' + bl, post=True)
                     finishedUploads.append(bl)
     for x in finishedUploads:
         try:
diff --git a/onionr/core.py b/onionr/core.py
index ebc35e51..53bf0b4c 100755
--- a/onionr/core.py
+++ b/onionr/core.py
@@ -28,6 +28,7 @@ from onionrusers import onionrusers
 from onionrstorage import removeblock, setdata
 import dbcreator, onionrstorage, serializeddata, subprocesspow
 from etc import onionrvalues, powchoice
+from onionrutils import localcommand
 
 class Core:
     def __init__(self, torPort=0):
@@ -433,8 +434,8 @@ class Core:
                 retData = False
             else:
                 # Tell the api server through localCommand to wait for the daemon to upload this block to make statistical analysis more difficult
-                if self._utils.localCommand('/ping', maxWait=10) == 'pong!':
-                    self._utils.localCommand('/waitforshare/' + retData, post=True, maxWait=5)
+                if localcommand.local_command(self, '/ping', maxWait=10) == 'pong!':
+                    localcommand.local_command(self, '/waitforshare/' + retData, post=True, maxWait=5)
                     self.daemonQueueAdd('uploadBlock', retData)
                 self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
                 self._utils.processBlockMetadata(retData)
@@ -450,7 +451,7 @@ class Core:
         '''
             Introduces our node into the network by telling X many nodes our HS address
         '''
-        if self._utils.localCommand('/ping', maxWait=10) == 'pong!':
+        if localcommand.local_command(self, '/ping', maxWait=10) == 'pong!':
             self.daemonQueueAdd('announceNode')
             logger.info('Introduction command will be processed.', terminal=True)
         else:
diff --git a/onionr/coredb/daemonqueue/__init__.py b/onionr/coredb/daemonqueue/__init__.py
index 8bd21cfd..a2f7a483 100644
--- a/onionr/coredb/daemonqueue/__init__.py
+++ b/onionr/coredb/daemonqueue/__init__.py
@@ -1,5 +1,7 @@
 import sqlite3, os
 import onionrevents as events
+from onionrutils import localcommand
+
 def daemon_queue(core_inst):
     '''
         Gives commands to the communication proccess/daemon by reading an sqlite3 database
@@ -55,7 +57,7 @@ def daemon_queue_get_response(core_inst, responseID=''):
         Get a response sent by communicator to the API, by requesting to the API
     '''
     assert len(responseID) > 0
-    resp = core_inst._utils.localCommand('queueResponse/' + responseID)
+    resp = localcommand.local_command(core_inst, 'queueResponse/' + responseID)
     return resp
 
 def clear_daemon_queue(core_inst):
diff --git a/onionr/coredb/keydb/addkeys.py b/onionr/coredb/keydb/addkeys.py
index ac219b1a..716581cd 100644
--- a/onionr/coredb/keydb/addkeys.py
+++ b/onionr/coredb/keydb/addkeys.py
@@ -1,5 +1,7 @@
 import sqlite3
-import onionrevents as events, config
+import onionrevents as events
+from onionrutils import stringvalidators
+
 def add_peer(core_inst, peerID, name=''):
     '''
         Adds a public key to the key database (misleading function name)
@@ -40,8 +42,8 @@ def add_address(core_inst, address):
 
     if type(address) is None or len(address) == 0:
         return False
-    if core_inst._utils.validateID(address):
-        if address == config.get('i2p.ownAddr', None) or address == core_inst.hsAddress:
+    if stringvalidators.validate_transport(address):
+        if address == core_inst.config.get('i2p.ownAddr', None) or address == core_inst.hsAddress:
             return False
         conn = sqlite3.connect(core_inst.addressDB, timeout=30)
         c = conn.cursor()
diff --git a/onionr/coredb/keydb/removekeys.py b/onionr/coredb/keydb/removekeys.py
index 10f44a1b..ce4ba5fc 100644
--- a/onionr/coredb/keydb/removekeys.py
+++ b/onionr/coredb/keydb/removekeys.py
@@ -1,11 +1,13 @@
 import sqlite3
 import onionrevents as events
+from onionrutils import stringvalidators
+
 def remove_address(core_inst, address):
     '''
         Remove an address from the address database
     '''
 
-    if core_inst._utils.validateID(address):
+    if stringvalidators.validate_transport(address):
         conn = sqlite3.connect(core_inst.addressDB, timeout=30)
         c = conn.cursor()
         t = (address,)
diff --git a/onionr/httpapi/miscpublicapi/announce.py b/onionr/httpapi/miscpublicapi/announce.py
index 8a25b635..f17c8d83 100755
--- a/onionr/httpapi/miscpublicapi/announce.py
+++ b/onionr/httpapi/miscpublicapi/announce.py
@@ -21,6 +21,8 @@ import base64
 from flask import Response
 import logger
 from etc import onionrvalues
+from onionrutils import stringvalidators
+
 def handle_announce(clientAPI, request):
     '''
     accept announcement posts, validating POW
@@ -52,7 +54,7 @@ def handle_announce(clientAPI, request):
                 pass
             if powHash.startswith('0' * onionrvalues.OnionrValues().announce_pow):
                 newNode = clientAPI._core._utils.bytesToStr(newNode)
-                if clientAPI._core._utils.validateID(newNode) and not newNode in clientAPI._core.onionrInst.communicatorInst.newPeers:
+                if stringvalidators.validate_transport(newNode) and not newNode in clientAPI._core.onionrInst.communicatorInst.newPeers:
                     clientAPI._core.onionrInst.communicatorInst.newPeers.append(newNode)
                     resp = 'Success'
             else:
diff --git a/onionr/onionrcommands/__init__.py b/onionr/onionrcommands/__init__.py
index 3a50d59d..59c9d6bc 100755
--- a/onionr/onionrcommands/__init__.py
+++ b/onionr/onionrcommands/__init__.py
@@ -22,6 +22,7 @@ import webbrowser, sys
 import logger
 from . import pubkeymanager, onionrstatistics, daemonlaunch, filecommands, plugincommands, keyadders
 from . import banblocks, exportblocks, openwebinterface, resettor
+from onionrutils import importnewblocks
 
 def show_help(o_inst, command):
 
@@ -110,8 +111,8 @@ def get_commands(onionr_inst):
     'listconn': onionr_inst.listConn,
     'list-conn': onionr_inst.listConn,
 
-    'import-blocks': onionr_inst.onionrUtils.importNewBlocks,
-    'importblocks': onionr_inst.onionrUtils.importNewBlocks,
+    'import-blocks': importnewblocks.import_new_blocks,
+    'importblocks': importnewblocks.import_new_blocks,
 
     'introduce': onionr_inst.onionrCore.introduceNode,
     'pex': onionr_inst.doPEX,
diff --git a/onionr/onionrcommands/daemonlaunch.py b/onionr/onionrcommands/daemonlaunch.py
index 1aaf7230..28202c10 100755
--- a/onionr/onionrcommands/daemonlaunch.py
+++ b/onionr/onionrcommands/daemonlaunch.py
@@ -23,9 +23,10 @@ from threading import Thread
 import onionr, api, logger, communicator
 import onionrevents as events
 from netcontroller import NetController
+from onionrutils import localcommand
 
 def _proper_shutdown(o_inst):
-    o_inst.onionrUtils.localCommand('shutdown')
+    localcommand.local_command(o_inst.onionrCore, 'shutdown')
     sys.exit(1)
 
 def daemon(o_inst):
@@ -63,7 +64,7 @@ def daemon(o_inst):
     net = NetController(o_inst.onionrCore.config.get('client.public.port', 59497), apiServerIP=apiHost)
     logger.info('Tor is starting...', terminal=True)
     if not net.startTor():
-        o_inst.onionrUtils.localCommand('shutdown')
+        localcommand.local_command(o_inst.onionrCore, 'shutdown')
         sys.exit(1)
     if len(net.myID) > 0 and o_inst.onionrCore.config.get('general.security_level', 1) == 0:
         logger.debug('Started .onion service: %s' % (logger.colors.underline + net.myID))
@@ -103,10 +104,10 @@ def daemon(o_inst):
 
     signal.signal(signal.SIGINT, _ignore_sigint)
     o_inst.onionrCore.daemonQueueAdd('shutdown')
-    o_inst.onionrUtils.localCommand('shutdown')
+    localcommand.local_command(o_inst.onionrCore, 'shutdown')
 
     net.killTor()
-    time.sleep(8) # Time to allow threads to finish, if not any "daemon" threads will be slaughtered http://docs.python.org/library/threading.html#threading.Thread.daemon
+    time.sleep(5) # Time to allow threads to finish, if not any "daemon" threads will be slaughtered http://docs.python.org/library/threading.html#threading.Thread.daemon
     o_inst.deleteRunFiles()
     return
 
diff --git a/onionr/onionrcommands/onionrstatistics.py b/onionr/onionrcommands/onionrstatistics.py
index a28fb7db..2f50c6e7 100755
--- a/onionr/onionrcommands/onionrstatistics.py
+++ b/onionr/onionrcommands/onionrstatistics.py
@@ -21,6 +21,7 @@ import os, uuid, time
 import logger, onionrutils
 from onionrblockapi import Block
 import onionr
+from onionrutils import checkcommunicator
 
 def show_stats(o_inst):
     try:
@@ -29,7 +30,7 @@ def show_stats(o_inst):
         signedBlocks = len(Block.getBlocks(signed = True))
         messages = {
             # info about local client
-            'Onionr Daemon Status' : ((logger.colors.fg.green + 'Online') if o_inst.onionrUtils.isCommunicatorRunning(timeout = 9) else logger.colors.fg.red + 'Offline'),
+            'Onionr Daemon Status' : ((logger.colors.fg.green + 'Online') if checkcommunicator.is_communicator_running(o_inst.onionrCore, timeout = 9) else logger.colors.fg.red + 'Offline'),
 
             # file and folder size stats
             'div1' : True, # this creates a solid line across the screen, a div
diff --git a/onionr/onionrcommands/openwebinterface.py b/onionr/onionrcommands/openwebinterface.py
index 8ce71744..42c80b57 100755
--- a/onionr/onionrcommands/openwebinterface.py
+++ b/onionr/onionrcommands/openwebinterface.py
@@ -19,9 +19,10 @@
 '''
 import webbrowser
 import logger
+from onionrutils import getclientapiserver
 def open_home(o_inst):
     try:
-        url = o_inst.onionrUtils.getClientAPIServer()
+        url = getclientapiserver.get_client_API_server(o_inst.onionrCore)
     except FileNotFoundError:
         logger.error('Onionr seems to not be running (could not get api host)', terminal=True)
     else:
diff --git a/onionr/onionrcommands/resettor.py b/onionr/onionrcommands/resettor.py
index cd7c51d2..e327bccc 100755
--- a/onionr/onionrcommands/resettor.py
+++ b/onionr/onionrcommands/resettor.py
@@ -19,11 +19,13 @@
 '''
 import os, shutil
 import logger, core
+from onionrutils import localcommand
+
 def reset_tor():
     c = core.Core()
     tor_dir = c.dataDir + 'tordata'
     if os.path.exists(tor_dir):
-        if c._utils.localCommand('/ping') == 'pong!':
+        if localcommand.local_command(c, '/ping') == 'pong!':
             logger.warn('Cannot delete Tor data while Onionr is running', terminal=True)
         else:
             shutil.rmtree(tor_dir)
\ No newline at end of file
diff --git a/onionr/onionrpluginapi.py b/onionr/onionrpluginapi.py
index 78c8a008..24ba4f12 100755
--- a/onionr/onionrpluginapi.py
+++ b/onionr/onionrpluginapi.py
@@ -19,6 +19,7 @@
 '''
 
 import onionrplugins, core as onionrcore, logger
+from onionrutils import localcommand
 
 class DaemonAPI:
     def __init__(self, pluginapi):
@@ -40,7 +41,7 @@ class DaemonAPI:
         return
 
     def local_command(self, command):
-        return self.pluginapi.get_utils().localCommand(self, command)
+        return localcommand.local_command(self.pluginapi.get_core(), command)
 
     def queue_pop(self):
         return self.get_core().daemonQueue()
diff --git a/onionr/onionrservices/__init__.py b/onionr/onionrservices/__init__.py
index 2792f7e3..1c23e5fb 100755
--- a/onionr/onionrservices/__init__.py
+++ b/onionr/onionrservices/__init__.py
@@ -21,6 +21,7 @@ import time
 import stem
 import core
 from . import connectionserver, bootstrapservice
+from onionrutils import stringvalidators
 
 class OnionrServices:
     '''
@@ -39,7 +40,7 @@ class OnionrServices:
             When a client wants to connect, contact their bootstrap address and tell them our
             ephemeral address for our service by creating a new ConnectionServer instance
         '''
-        assert self._core._utils.validateID(address)
+        assert stringvalidators.validate_transport(address)
         BOOTSTRAP_TRIES = 10 # How many times to attempt contacting the bootstrap server
         TRY_WAIT = 3 # Seconds to wait before trying bootstrap again
         # HTTP is fine because .onion/i2p is encrypted/authenticated
diff --git a/onionr/onionrservices/bootstrapservice.py b/onionr/onionrservices/bootstrapservice.py
index 077b4657..e06300e6 100755
--- a/onionr/onionrservices/bootstrapservice.py
+++ b/onionr/onionrservices/bootstrapservice.py
@@ -24,6 +24,7 @@ from flask import Flask, Response
 import core
 from netcontroller import getOpenPort
 from . import httpheaders
+from onionrutils import stringvalidators
 
 def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=300):
     '''
@@ -61,7 +62,7 @@ def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=300):
 
     @bootstrap_app.route('/bs/<address>', methods=['POST'])
     def get_bootstrap(address):
-        if core_inst._utils.validateID(address + '.onion'):
+        if stringvalidators.validate_transport(address + '.onion'):
             # Set the bootstrap address then close the server
             bootstrap_address = address + '.onion'
             core_inst.keyStore.put(bs_id, bootstrap_address)
diff --git a/onionr/onionrutils/__init__.py b/onionr/onionrutils/__init__.py
index a7efbae7..23907a00 100755
--- a/onionr/onionrutils/__init__.py
+++ b/onionr/onionrutils/__init__.py
@@ -29,6 +29,7 @@ import storagecounter
 from etc import pgpwords, onionrvalues
 from onionrusers import onionrusers 
 from . import localcommand, blockmetadata, validatemetadata, basicrequests
+from . import stringvalidators
 
 config.reload()
 class OnionrUtils:
@@ -50,23 +51,6 @@ class OnionrUtils:
         '''
         epoch = self.getEpoch()
         return epoch - (epoch % roundS)
-    
-    def getClientAPIServer(self):
-        retData = ''
-        try:
-            with open(self._core.privateApiHostFile, 'r') as host:
-                hostname = host.read()
-        except FileNotFoundError:
-            raise FileNotFoundError
-        else:
-            retData += '%s:%s' % (hostname, config.get('client.client.port'))
-        return retData
-
-    def localCommand(self, command, data='', silent = True, post=False, postData = {}, maxWait=20):
-        '''
-            Send a command to the local http API server, securely. Intended for local clients, DO NOT USE for remote peers.
-        '''
-        return localcommand.local_command(self, command, data, silent, post, postData, maxWait)
 
     def getHumanReadableID(self, pub=''):
         '''gets a human readable ID from a public key'''
@@ -132,19 +116,7 @@ class OnionrUtils:
         '''
             Validate if a string is a valid hash hex digest (does not compare, just checks length and charset)
         '''
-        retVal = True
-        if data == False or data == True:
-            return False
-        data = data.strip()
-        if len(data) != length:
-            retVal = False
-        else:
-            try:
-                int(data, 16)
-            except ValueError:
-                retVal = False
-
-        return retVal
+        return stringvalidators.validate_hash(self, data, length)
 
     def validateMetadata(self, metadata, blockData):
         '''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string'''
@@ -154,129 +126,7 @@ class OnionrUtils:
         '''
             Validate if a string is a valid base32 encoded Ed25519 key
         '''
-        if type(key) is type(None):
-            return False
-        # Accept keys that have no = padding
-        key = unpaddedbase32.repad(self.strToBytes(key))
-
-        retVal = False
-        try:
-            nacl.signing.SigningKey(seed=key, encoder=nacl.encoding.Base32Encoder)
-        except nacl.exceptions.ValueError:
-            pass
-        except base64.binascii.Error as err:
-            pass
-        else:
-            retVal = True
-        return retVal
-
-    @staticmethod
-    def validateID(id):
-        '''
-            Validate if an address is a valid tor or i2p hidden service
-        '''
-        try:
-            idLength = len(id)
-            retVal = True
-            idNoDomain = ''
-            peerType = ''
-            # i2p b32 addresses are 60 characters long (including .b32.i2p)
-            if idLength == 60:
-                peerType = 'i2p'
-                if not id.endswith('.b32.i2p'):
-                    retVal = False
-                else:
-                    idNoDomain = id.split('.b32.i2p')[0]
-            # Onion v2's are 22 (including .onion), v3's are 62 with .onion
-            elif idLength == 22 or idLength == 62:
-                peerType = 'onion'
-                if not id.endswith('.onion'):
-                    retVal = False
-                else:
-                    idNoDomain = id.split('.onion')[0]
-            else:
-                retVal = False
-            if retVal:
-                if peerType == 'i2p':
-                    try:
-                        id.split('.b32.i2p')[2]
-                    except:
-                        pass
-                    else:
-                        retVal = False
-                elif peerType == 'onion':
-                    try:
-                        id.split('.onion')[2]
-                    except:
-                        pass
-                    else:
-                        retVal = False
-                if not idNoDomain.isalnum():
-                    retVal = False
-
-                # Validate address is valid base32 (when capitalized and minus extension); v2/v3 onions and .b32.i2p use base32
-                for x in idNoDomain.upper():
-                    if x not in string.ascii_uppercase and x not in '234567':
-                        retVal = False
-
-            return retVal
-        except:
-            return False
-
-    @staticmethod
-    def isIntegerString(data):
-        '''Check if a string is a valid base10 integer (also returns true if already an int)'''
-        try:
-            int(data)
-        except (ValueError, TypeError) as e:
-            return False
-        else:
-            return True
-
-    def isCommunicatorRunning(self, timeout = 5, interval = 0.1):
-        try:
-            runcheck_file = self._core.dataDir + '.runcheck'
-
-            if not os.path.isfile(runcheck_file):
-                open(runcheck_file, 'w+').close()
-
-            # self._core.daemonQueueAdd('runCheck') # deprecated
-            starttime = time.time()
-
-            while True:
-                time.sleep(interval)
-
-                if not os.path.isfile(runcheck_file):
-                    return True
-                elif time.time() - starttime >= timeout:
-                    return False
-        except:
-            return False
-
-    def importNewBlocks(self, scanDir=''):
-        '''
-            This function is intended to scan for new blocks ON THE DISK and import them
-        '''
-        blockList = self._core.getBlockList()
-        exist = False
-        if scanDir == '':
-            scanDir = self._core.blockDataLocation
-        if not scanDir.endswith('/'):
-            scanDir += '/'
-        for block in glob.glob(scanDir + "*.dat"):
-            if block.replace(scanDir, '').replace('.dat', '') not in blockList:
-                exist = True
-                logger.info('Found new block on dist %s' % block)
-                with open(block, 'rb') as newBlock:
-                    block = block.replace(scanDir, '').replace('.dat', '')
-                    if self._core._crypto.sha3Hash(newBlock.read()) == block.replace('.dat', ''):
-                        self._core.addToBlockDB(block.replace('.dat', ''), dataSaved=True)
-                        logger.info('Imported block %s.' % block)
-                        self._core._utils.processBlockMetadata(block)
-                    else:
-                        logger.warn('Failed to verify hash for %s' % block)
-        if not exist:
-            logger.info('No blocks found to import')
+        return stringvalidators.validate_pub_key(self, key)
 
     def getEpoch(self):
         '''returns epoch'''
diff --git a/onionr/onionrutils/checkcommunicator.py b/onionr/onionrutils/checkcommunicator.py
new file mode 100644
index 00000000..07f8dc1a
--- /dev/null
+++ b/onionr/onionrutils/checkcommunicator.py
@@ -0,0 +1,19 @@
+import time, os
+def is_communicator_running(core_inst, timeout = 5, interval = 0.1):
+    try:
+        runcheck_file = core_inst.dataDir + '.runcheck'
+
+        if not os.path.isfile(runcheck_file):
+            open(runcheck_file, 'w+').close()
+
+        starttime = time.time()
+
+        while True:
+            time.sleep(interval)
+
+            if not os.path.isfile(runcheck_file):
+                return True
+            elif time.time() - starttime >= timeout:
+                return False
+    except:
+        return False
\ No newline at end of file
diff --git a/onionr/onionrutils/getclientapiserver.py b/onionr/onionrutils/getclientapiserver.py
new file mode 100644
index 00000000..23020f66
--- /dev/null
+++ b/onionr/onionrutils/getclientapiserver.py
@@ -0,0 +1,10 @@
+def get_client_API_server(core_inst):
+    retData = ''
+    try:
+        with open(core_inst.privateApiHostFile, 'r') as host:
+            hostname = host.read()
+    except FileNotFoundError:
+        raise FileNotFoundError
+    else:
+        retData += '%s:%s' % (hostname, core_inst.config.get('client.client.port'))
+    return retData
\ No newline at end of file
diff --git a/onionr/onionrutils/importnewblocks.py b/onionr/onionrutils/importnewblocks.py
new file mode 100644
index 00000000..a2c1e518
--- /dev/null
+++ b/onionr/onionrutils/importnewblocks.py
@@ -0,0 +1,28 @@
+import glob
+import logger, core
+def import_new_blocks(core_inst=None, scanDir=''):
+    '''
+        This function is intended to scan for new blocks ON THE DISK and import them
+    '''
+    if core_inst is None:
+        core_inst = core.Core()
+    blockList = core_inst.getBlockList()
+    exist = False
+    if scanDir == '':
+        scanDir = core_inst.blockDataLocation
+    if not scanDir.endswith('/'):
+        scanDir += '/'
+    for block in glob.glob(scanDir + "*.dat"):
+        if block.replace(scanDir, '').replace('.dat', '') not in blockList:
+            exist = True
+            logger.info('Found new block on dist %s' % block)
+            with open(block, 'rb') as newBlock:
+                block = block.replace(scanDir, '').replace('.dat', '')
+                if core_inst._crypto.sha3Hash(newBlock.read()) == block.replace('.dat', ''):
+                    core_inst.addToBlockDB(block.replace('.dat', ''), dataSaved=True)
+                    logger.info('Imported block %s.' % block)
+                    core_inst._utils.processBlockMetadata(block)
+                else:
+                    logger.warn('Failed to verify hash for %s' % block)
+    if not exist:
+        logger.info('No blocks found to import')
\ No newline at end of file
diff --git a/onionr/onionrutils/localcommand.py b/onionr/onionrutils/localcommand.py
index fff050c8..90b0d7ac 100644
--- a/onionr/onionrutils/localcommand.py
+++ b/onionr/onionrutils/localcommand.py
@@ -1,6 +1,7 @@
 import urllib, requests, time
 import logger
-def local_command(utils_inst, command, data='', silent = True, post=False, postData = {}, maxWait=20):
+from onionrutils import getclientapiserver
+def local_command(core_inst, command, data='', silent = True, post=False, postData = {}, maxWait=20):
     '''
         Send a command to the local http API server, securely. Intended for local clients, DO NOT USE for remote peers.
     '''
@@ -9,7 +10,7 @@ def local_command(utils_inst, command, data='', silent = True, post=False, postD
     waited = 0
     while hostname == '':
         try:
-            hostname = utils_inst.getClientAPIServer()
+            hostname = getclientapiserver.get_client_API_server(core_inst)
         except FileNotFoundError:
             time.sleep(1)
             waited += 1
diff --git a/onionr/onionrutils/stringvalidators.py b/onionr/onionrutils/stringvalidators.py
new file mode 100644
index 00000000..49f2c33a
--- /dev/null
+++ b/onionr/onionrutils/stringvalidators.py
@@ -0,0 +1,97 @@
+import base64, string
+import unpaddedbase32, nacl.signing, nacl.encoding
+def validate_hash(utils_inst, data, length=64):
+    '''
+        Validate if a string is a valid hash hex digest (does not compare, just checks length and charset)
+    '''
+    retVal = True
+    if data == False or data == True:
+        return False
+    data = data.strip()
+    if len(data) != length:
+        retVal = False
+    else:
+        try:
+            int(data, 16)
+        except ValueError:
+            retVal = False
+
+    return retVal
+
+def validate_pub_key(utils_inst, key):
+    '''
+        Validate if a string is a valid base32 encoded Ed25519 key
+    '''
+    if type(key) is type(None):
+        return False
+    # Accept keys that have no = padding
+    key = unpaddedbase32.repad(utils_inst.strToBytes(key))
+
+    retVal = False
+    try:
+        nacl.signing.SigningKey(seed=key, encoder=nacl.encoding.Base32Encoder)
+    except nacl.exceptions.ValueError:
+        pass
+    except base64.binascii.Error as err:
+        pass
+    else:
+        retVal = True
+    return retVal
+
+def validate_transport(id):
+    try:
+        idLength = len(id)
+        retVal = True
+        idNoDomain = ''
+        peerType = ''
+        # i2p b32 addresses are 60 characters long (including .b32.i2p)
+        if idLength == 60:
+            peerType = 'i2p'
+            if not id.endswith('.b32.i2p'):
+                retVal = False
+            else:
+                idNoDomain = id.split('.b32.i2p')[0]
+        # Onion v2's are 22 (including .onion), v3's are 62 with .onion
+        elif idLength == 22 or idLength == 62:
+            peerType = 'onion'
+            if not id.endswith('.onion'):
+                retVal = False
+            else:
+                idNoDomain = id.split('.onion')[0]
+        else:
+            retVal = False
+        if retVal:
+            if peerType == 'i2p':
+                try:
+                    id.split('.b32.i2p')[2]
+                except:
+                    pass
+                else:
+                    retVal = False
+            elif peerType == 'onion':
+                try:
+                    id.split('.onion')[2]
+                except:
+                    pass
+                else:
+                    retVal = False
+            if not idNoDomain.isalnum():
+                retVal = False
+
+            # Validate address is valid base32 (when capitalized and minus extension); v2/v3 onions and .b32.i2p use base32
+            for x in idNoDomain.upper():
+                if x not in string.ascii_uppercase and x not in '234567':
+                    retVal = False
+
+        return retVal
+    except Exception as e:
+        return False
+
+def is_integer_string(data):
+    '''Check if a string is a valid base10 integer (also returns true if already an int)'''
+    try:
+        int(data)
+    except (ValueError, TypeError) as e:
+        return False
+    else:
+        return True
diff --git a/onionr/onionrutils/validatemetadata.py b/onionr/onionrutils/validatemetadata.py
index c4bcc4a8..8feb9859 100644
--- a/onionr/onionrutils/validatemetadata.py
+++ b/onionr/onionrutils/validatemetadata.py
@@ -1,6 +1,7 @@
 import json
 import logger, onionrexceptions
 from etc import onionrvalues
+from onionrutils import stringvalidators
 def validate_metadata(utils_inst, metadata, blockData):
     '''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string'''
     # TODO, make this check sane sizes
@@ -15,7 +16,7 @@ def validate_metadata(utils_inst, metadata, blockData):
             pass
 
     # Validate metadata dict for invalid keys to sizes that are too large
-    maxAge = utils_inst._coreconfig.get("general.max_block_age", onionrvalues.OnionrValues().default_expire)
+    maxAge = utils_inst._core.config.get("general.max_block_age", onionrvalues.OnionrValues().default_expire)
     if type(metadata) is dict:
         for i in metadata:
             try:
@@ -33,7 +34,7 @@ def validate_metadata(utils_inst, metadata, blockData):
                     logger.warn('Block metadata key ' + i + ' exceeded maximum size')
                     break
             if i == 'time':
-                if not utils_inst.isIntegerString(metadata[i]):
+                if not stringvalidators.is_integer_string(metadata[i]):
                     logger.warn('Block metadata time stamp is not integer string or int')
                     break
                 isFuture = (metadata[i] - utils_inst.getEpoch())
diff --git a/onionr/static-data/default-plugins/cliui/main.py b/onionr/static-data/default-plugins/cliui/main.py
index f6a0b906..c65917c8 100755
--- a/onionr/static-data/default-plugins/cliui/main.py
+++ b/onionr/static-data/default-plugins/cliui/main.py
@@ -23,6 +23,7 @@ import threading, time, uuid, subprocess, sys
 import config, logger
 from onionrblockapi import Block
 import onionrplugins
+from onionrutils import localcommand
 
 plugin_name = 'cliui'
 PLUGIN_VERSION = '0.0.1'
@@ -48,7 +49,7 @@ class OnionrCLIUI:
     
     def isRunning(self):
         while not self.shutdown:
-            if self.myCore._utils.localCommand('ping', maxWait=5) == 'pong!':
+            if localcommand.local_command(self.myCore, 'ping', maxWait=5) == 'pong!':
                 self.running = 'Yes'
             else:
                 self.running = 'No'
diff --git a/onionr/static-data/default-plugins/esoteric/peerserver.py b/onionr/static-data/default-plugins/esoteric/peerserver.py
index b3ae423e..f48ba0be 100755
--- a/onionr/static-data/default-plugins/esoteric/peerserver.py
+++ b/onionr/static-data/default-plugins/esoteric/peerserver.py
@@ -19,6 +19,7 @@
 '''
 import sys, os, json
 import core
+from onionrutils import localcommand
 from flask import Response, request, redirect, Blueprint, abort, g
 sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
 direct_blueprint = Blueprint('esoteric', __name__)
@@ -47,9 +48,9 @@ def sendto():
         msg = ''
     else:
         msg = json.dumps(msg)
-        core_inst._utils.localCommand('/esoteric/addrec/%s' % (g.peer,), post=True, postData=msg)
+        localcommand.local_command(core_inst, '/esoteric/addrec/%s' % (g.peer,), post=True, postData=msg)
     return Response('success')
 
 @direct_blueprint.route('/esoteric/poll')
 def poll_chat():
-    return Response(core_inst._utils.localCommand('/esoteric/gets/%s' % (g.peer,)))
\ No newline at end of file
+    return Response(localcommand.local_command(core_inst, '/esoteric/gets/%s' % (g.peer,)))
\ No newline at end of file
diff --git a/onionr/static-data/default-plugins/pluginmanager/main.py b/onionr/static-data/default-plugins/pluginmanager/main.py
index d4feb69d..2f261bbf 100755
--- a/onionr/static-data/default-plugins/pluginmanager/main.py
+++ b/onionr/static-data/default-plugins/pluginmanager/main.py
@@ -22,6 +22,7 @@
 import logger, config
 import os, sys, json, time, random, shutil, base64, getpass, datetime, re
 from onionrblockapi import Block
+from onionrutils import importnewblocks
 
 plugin_name = 'pluginmanager'
 
@@ -236,7 +237,7 @@ def pluginToBlock(plugin, import_block = True):
             # hash = pluginapi.get_core().insertBlock(, header = 'plugin', sign = True)
 
             if import_block:
-                pluginapi.get_utils().importNewBlocks()
+                importnewblocks.import_new_blocks(pluginapi.get_core())
 
             return hash
         else:
diff --git a/onionr/tests/test_stringvalidations.py b/onionr/tests/test_stringvalidations.py
index caac34df..4eb7f113 100755
--- a/onionr/tests/test_stringvalidations.py
+++ b/onionr/tests/test_stringvalidations.py
@@ -6,6 +6,7 @@ TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
 print("Test directory:", TEST_DIR)
 os.environ["ONIONR_HOME"] = TEST_DIR
 import core, onionr
+from onionrutils import stringvalidators
 
 core.Core()
 
@@ -13,7 +14,6 @@ class OnionrValidations(unittest.TestCase):
     
     def test_peer_validator(self):
         # Test hidden service domain validities
-        c = core.Core()
         valid = ['facebookcorewwwi.onion', 'vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd.onion', 
         '5bvb5ncnfr4dlsfriwczpzcvo65kn7fnnlnt2ln7qvhzna2xaldq.b32.i2p']
 
@@ -21,11 +21,11 @@ class OnionrValidations(unittest.TestCase):
 
         for x in valid:
             print('testing', x)
-            self.assertTrue(c._utils.validateID(x))
+            self.assertTrue(stringvalidators.validate_transport(x))
         
         for x in invalid:
             print('testing', x)
-            self.assertFalse(c._utils.validateID(x))
+            self.assertFalse(stringvalidators.validate_transport(x))
     
     def test_pubkey_validator(self):
         # Test ed25519 public key validity
@@ -44,14 +44,13 @@ class OnionrValidations(unittest.TestCase):
     def test_integer_string(self):
         valid = ["1", "100", 100, "-5", -5]
         invalid = ['test', "1d3434", "1e100", None]
-        c = core.Core()
 
         for x in valid:
             #print('testing', x)
-            self.assertTrue(c._utils.isIntegerString(x))   
+            self.assertTrue(stringvalidators.is_integer_string(x))   
         
         for x in invalid:
             #print('testing', x)
-            self.assertFalse(c._utils.isIntegerString(x))
+            self.assertFalse(stringvalidators.is_integer_string(x))
     
 unittest.main()
\ No newline at end of file

From 909c002dc4b38f87690b4d070b30a2a6a4ccb32c Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Tue, 25 Jun 2019 03:21:36 -0500
Subject: [PATCH 13/29] more progress in removing onionrutils class

---
 onionr/api.py                                 |  13 +--
 onionr/blockimporter.py                       |   9 +-
 onionr/communicatorutils/downloadblocks.py    |   5 +-
 onionr/communicatorutils/servicecreator.py    |   2 +-
 onionr/core.py                                |  15 +--
 onionr/coredb/blockmetadb/add.py              |   3 +-
 onionr/coredb/keydb/addkeys.py                |   2 +-
 onionr/httpapi/miscpublicapi/getblocks.py     |   8 +-
 onionr/netcontroller.py                       |   1 -
 onionr/onionr.py                              |  11 +-
 onionr/onionrblockapi.py                      |   3 +-
 onionr/onionrcommands/keyadders.py            |   2 +-
 onionr/onionrcommands/onionrstatistics.py     |   4 +-
 onionr/onionrcommands/pubkeymanager.py        |   5 +-
 onionr/onionrcrypto.py                        |  21 ++--
 onionr/onionrproofs.py                        |   7 +-
 onionr/onionrservices/bootstrapservice.py     |   2 +-
 onionr/onionrservices/connectionserver.py     |   3 +-
 onionr/onionrstorage/__init__.py              |   3 +-
 onionr/onionrusers/contactmanager.py          |   3 +-
 onionr/onionrusers/onionrusers.py             |  12 +-
 onionr/onionrutils/__init__.py                | 106 ++++--------------
 onionr/onionrutils/basicrequests.py           |   2 +-
 onionr/onionrutils/blockmetadata.py           |   9 +-
 onionr/onionrutils/bytesconverter.py          |  13 +++
 onionr/onionrutils/epoch.py                   |  11 ++
 onionr/onionrutils/importnewblocks.py         |   3 +-
 onionr/onionrutils/mnemonickeys.py            |   8 ++
 onionr/onionrutils/stringvalidators.py        |   6 +-
 onionr/onionrutils/validatemetadata.py        |  20 ++--
 .../default-plugins/encrypt/main.py           |   3 +-
 .../default-plugins/esoteric/main.py          |   3 +-
 .../default-plugins/metadataprocessor/main.py |   3 +-
 .../default-plugins/pluginmanager/main.py     |   8 +-
 .../static-data/default-plugins/pms/main.py   |   3 +-
 onionr/subprocesspow.py                       |   5 +-
 onionr/tests/test_highlevelcrypto.py          |  16 +--
 37 files changed, 164 insertions(+), 189 deletions(-)
 mode change 100755 => 100644 onionr/onionrutils/__init__.py
 create mode 100644 onionr/onionrutils/bytesconverter.py
 create mode 100644 onionr/onionrutils/epoch.py
 create mode 100644 onionr/onionrutils/mnemonickeys.py

diff --git a/onionr/api.py b/onionr/api.py
index ab1eeb59..7d8bf671 100755
--- a/onionr/api.py
+++ b/onionr/api.py
@@ -23,8 +23,7 @@ from gevent import Timeout
 import flask
 from flask import request, Response, abort, send_from_directory
 import core
-from onionrblockapi import Block
-import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config
+import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config, onionrblockapi
 import httpapi
 from httpapi import friendsapi, profilesapi, configapi, miscpublicapi
 from onionrservices import httpheaders
@@ -337,7 +336,7 @@ class API:
             resp = ''
             if self._core._utils.validateHash(name):
                 try:
-                    resp = Block(name, decrypt=True).bcontent
+                    resp = onionrblockapi.Block(name, decrypt=True).bcontent
                 except TypeError:
                     pass
             else:
@@ -374,7 +373,7 @@ class API:
             resp = 'Not Found'
             if self._core._utils.validateHash(bHash):
                 try:
-                    resp = Block(bHash).bcontent
+                    resp = onionrblockapi.Block(bHash).bcontent
                 except onionrexceptions.NoDataAvailable:
                     abort(404)
                 except TypeError:
@@ -505,7 +504,7 @@ class API:
     
     def getBlockData(self, bHash, decrypt=False, raw=False, headerOnly=False):
         assert self._core._utils.validateHash(bHash)
-        bl = Block(bHash, core=self._core)
+        bl = onionrblockapi.Block(bHash, core=self._core)
         if decrypt:
             bl.decrypt()
             if bl.isEncrypted and not bl.decrypted:
@@ -521,8 +520,8 @@ class API:
                         pass
             else:
                 validSig = False
-                signer = self._core._utils.bytesToStr(bl.signer)
-                if bl.isSigned() and self._core._utils.validatePubKey(signer) and bl.isSigner(signer):
+                signer = onionrutils.bytes_to_str(bl.signer)
+                if bl.isSigned() and onionrutils.stringvalidators.validate_pub_key(signer) and bl.isSigner(signer):
                     validSig = True                    
                 bl.bheader['validSig'] = validSig
                 bl.bheader['meta'] = ''
diff --git a/onionr/blockimporter.py b/onionr/blockimporter.py
index ccfc77db..d5e02e7d 100755
--- a/onionr/blockimporter.py
+++ b/onionr/blockimporter.py
@@ -18,6 +18,7 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 import core, onionrexceptions, logger
+from onionrutils import validatemetadata, blockmetadata
 def importBlockFromData(content, coreInst):
     retData = False
 
@@ -34,17 +35,17 @@ def importBlockFromData(content, coreInst):
     except AttributeError:
         pass
 
-    metas = coreInst._utils.getBlockMetadataFromData(content) # returns tuple(metadata, meta), meta is also in metadata
+    metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
     metadata = metas[0]
-    if coreInst._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid
+    if validatemetadata(metadata, metas[2]): # check if metadata is valid
         if coreInst._crypto.verifyPow(content): # check if POW is enough/correct
-            logger.info('Block passed proof, saving.')
+            logger.info('Block passed proof, saving.', terminal=True)
             try:
                 blockHash = coreInst.setData(content)
             except onionrexceptions.DiskAllocationReached:
                 pass
             else:
                 coreInst.addToBlockDB(blockHash, dataSaved=True)
-                coreInst._utils.processBlockMetadata(blockHash) # caches block metadata values to block database
+                blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database
                 retData = True
     return retData
\ No newline at end of file
diff --git a/onionr/communicatorutils/downloadblocks.py b/onionr/communicatorutils/downloadblocks.py
index cb72bd12..ccf121c5 100755
--- a/onionr/communicatorutils/downloadblocks.py
+++ b/onionr/communicatorutils/downloadblocks.py
@@ -19,6 +19,7 @@
 '''
 import communicator, onionrexceptions
 import logger, onionrpeers
+from onionrutils import blockmetadata
 
 def download_blocks_from_communicator(comm_inst):
     assert isinstance(comm_inst, communicator.OnionrCommunicatorDaemon)
@@ -72,7 +73,7 @@ def download_blocks_from_communicator(comm_inst):
                 pass
             if realHash == blockHash:
                 content = content.decode() # decode here because sha3Hash needs bytes above
-                metas = comm_inst._core._utils.getBlockMetadataFromData(content) # returns tuple(metadata, meta), meta is also in metadata
+                metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
                 metadata = metas[0]
                 if comm_inst._core._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid, and verify nonce
                     if comm_inst._core._crypto.verifyPow(content): # check if POW is enough/correct
@@ -84,7 +85,7 @@ def download_blocks_from_communicator(comm_inst):
                             removeFromQueue = False
                         else:
                             comm_inst._core.addToBlockDB(blockHash, dataSaved=True)
-                            comm_inst._core._utils.processBlockMetadata(blockHash) # caches block metadata values to block database
+                            blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database
                     else:
                         logger.warn('POW failed for block %s.' % blockHash)
                 else:
diff --git a/onionr/communicatorutils/servicecreator.py b/onionr/communicatorutils/servicecreator.py
index 94fc51c8..07d3adfb 100755
--- a/onionr/communicatorutils/servicecreator.py
+++ b/onionr/communicatorutils/servicecreator.py
@@ -32,7 +32,7 @@ def service_creator(daemon):
         if not b in daemon.active_services:
             bl = onionrblockapi.Block(b, core=core, decrypt=True)
             bs = utils.bytesToStr(bl.bcontent) + '.onion'
-            if utils.validatePubKey(bl.signer) and stringvalidators.validate_transport(bs):
+            if stringvalidators.validate_pub_key(bl.signer) and stringvalidators.validate_transport(bs):
                 signer = utils.bytesToStr(bl.signer)
                 daemon.active_services.append(b)
                 daemon.active_services.append(signer)
diff --git a/onionr/core.py b/onionr/core.py
index 53bf0b4c..96068a66 100755
--- a/onionr/core.py
+++ b/onionr/core.py
@@ -28,7 +28,8 @@ from onionrusers import onionrusers
 from onionrstorage import removeblock, setdata
 import dbcreator, onionrstorage, serializeddata, subprocesspow
 from etc import onionrvalues, powchoice
-from onionrutils import localcommand
+from onionrutils import localcommand, stringvalidators, bytesconverter, epoch
+from onionrutils import blockmetadata
 
 class Core:
     def __init__(self, torPort=0):
@@ -320,9 +321,9 @@ class Core:
         if type(data) is None:
             raise ValueError('Data cannot be none')
 
-        createTime = self._utils.getRoundedEpoch()
+        createTime = epoch.get_epoch()
 
-        dataNonce = self._utils.bytesToStr(self._crypto.sha3Hash(data))
+        dataNonce = bytesconverter.bytes_to_str(self._crypto.sha3Hash(data))
         try:
             with open(self.dataNonceFile, 'r') as nonces:
                 if dataNonce in nonces:
@@ -395,7 +396,7 @@ class Core:
             signature = self._crypto.symmetricEncrypt(signature, key=symKey, returnEncoded=True).decode()
             signer = self._crypto.symmetricEncrypt(signer, key=symKey, returnEncoded=True).decode()
         elif encryptType == 'asym':
-            if self._utils.validatePubKey(asymPeer):
+            if stringvalidators.validate_pub_key(asymPeer):
                 # Encrypt block data with forward secrecy key first, but not meta
                 jsonMeta = json.dumps(meta)
                 jsonMeta = self._crypto.pubKeyEncrypt(jsonMeta, asymPeer, encodedData=True).decode()
@@ -438,13 +439,13 @@ class Core:
                     localcommand.local_command(self, '/waitforshare/' + retData, post=True, maxWait=5)
                     self.daemonQueueAdd('uploadBlock', retData)
                 self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
-                self._utils.processBlockMetadata(retData)
+                blockmetadata.process_block_metadata(retData)
 
         if retData != False:
             if plaintextPeer == onionrvalues.DENIABLE_PEER_ADDRESS:
-                events.event('insertdeniable', {'content': plaintext, 'meta': plaintextMeta, 'hash': retData, 'peer': self._utils.bytesToStr(asymPeer)}, onionr = self.onionrInst, threaded = True)
+                events.event('insertdeniable', {'content': plaintext, 'meta': plaintextMeta, 'hash': retData, 'peer': bytesconverter.bytes_to_str(asymPeer)}, onionr = self.onionrInst, threaded = True)
             else:
-                events.event('insertblock', {'content': plaintext, 'meta': plaintextMeta, 'hash': retData, 'peer': self._utils.bytesToStr(asymPeer)}, onionr = self.onionrInst, threaded = True)
+                events.event('insertblock', {'content': plaintext, 'meta': plaintextMeta, 'hash': retData, 'peer': bytesconverter.bytes_to_str(asymPeer)}, onionr = self.onionrInst, threaded = True)
         return retData
 
     def introduceNode(self):
diff --git a/onionr/coredb/blockmetadb/add.py b/onionr/coredb/blockmetadb/add.py
index d307d351..69ac9b27 100644
--- a/onionr/coredb/blockmetadb/add.py
+++ b/onionr/coredb/blockmetadb/add.py
@@ -1,4 +1,5 @@
 import os, sqlite3
+import onionrutils
 def add_to_block_DB(core_inst, newHash, selfInsert=False, dataSaved=False):
     '''
         Add a hash value to the block db
@@ -8,7 +9,7 @@ def add_to_block_DB(core_inst, newHash, selfInsert=False, dataSaved=False):
 
     if not os.path.exists(core_inst.blockDB):
         raise Exception('Block db does not exist')
-    if core_inst._utils.hasBlock(newHash):
+    if onionrutils.has_block(core_inst, newHash):
         return
     conn = sqlite3.connect(core_inst.blockDB, timeout=30)
     c = conn.cursor()
diff --git a/onionr/coredb/keydb/addkeys.py b/onionr/coredb/keydb/addkeys.py
index 716581cd..20d7500a 100644
--- a/onionr/coredb/keydb/addkeys.py
+++ b/onionr/coredb/keydb/addkeys.py
@@ -10,7 +10,7 @@ def add_peer(core_inst, peerID, name=''):
         raise ValueError("specified id is already known")
 
     # This function simply adds a peer to the DB
-    if not core_inst._utils.validatePubKey(peerID):
+    if not stringvalidators.validate_pub_key(peerID):
         return False
 
     events.event('pubkey_add', data = {'key': peerID}, onionr = core_inst.onionrInst)
diff --git a/onionr/httpapi/miscpublicapi/getblocks.py b/onionr/httpapi/miscpublicapi/getblocks.py
index fa9dde14..c1e58e4f 100755
--- a/onionr/httpapi/miscpublicapi/getblocks.py
+++ b/onionr/httpapi/miscpublicapi/getblocks.py
@@ -18,12 +18,12 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 from flask import Response, abort
-import config
+import config, onionrutils
 def get_public_block_list(clientAPI, publicAPI, request):
     # Provide a list of our blocks, with a date offset
     dateAdjust = request.args.get('date')
     bList = clientAPI._core.getBlockList(dateRec=dateAdjust)
-    if config.get('general.hide_created_blocks', True):
+    if clientAPI._core.config.get('general.hide_created_blocks', True):
         for b in publicAPI.hideBlocks:
             if b in bList:
                 # Don't share blocks we created if they haven't been *uploaded* yet, makes it harder to find who created a block
@@ -34,14 +34,14 @@ def get_block_data(clientAPI, publicAPI, data):
     '''data is the block hash in hex'''
     resp = ''
     if clientAPI._utils.validateHash(data):
-        if not config.get('general.hide_created_blocks', True) or data not in publicAPI.hideBlocks:
+        if not clientAPI._core.config.get('general.hide_created_blocks', True) or data not in publicAPI.hideBlocks:
             if data in clientAPI._core.getBlockList():
                 block = clientAPI.getBlockData(data, raw=True)
                 try:
                     block = block.encode() # Encode in case data is binary
                 except AttributeError:
                     abort(404)
-                block = clientAPI._core._utils.strToBytes(block)
+                block = onionrutils.str_to_bytes(block)
                 resp = block
     if len(resp) == 0:
         abort(404)
diff --git a/onionr/netcontroller.py b/onionr/netcontroller.py
index b2d8237f..3d43db8c 100755
--- a/onionr/netcontroller.py
+++ b/onionr/netcontroller.py
@@ -20,7 +20,6 @@
 import subprocess, os, sys, time, signal, base64, socket
 from shutil import which
 import logger, config
-from onionrblockapi import Block
 config.reload()
 def getOpenPort():
     # taken from (but modified) https://stackoverflow.com/a/2838309 by https://stackoverflow.com/users/133374/albert ccy-by-sa-3 https://creativecommons.org/licenses/by-sa/3.0/
diff --git a/onionr/onionr.py b/onionr/onionr.py
index 4030aae1..74c0c335 100755
--- a/onionr/onionr.py
+++ b/onionr/onionr.py
@@ -21,6 +21,10 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 import sys
+ONIONR_TAGLINE = 'Private P2P Communication - GPLv3 - https://Onionr.net'
+ONIONR_VERSION = '0.0.0' # for debugging and stuff
+ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION)
+API_VERSION = '0' # increments of 1; only change when something fundamental about how the API works changes. This way other nodes know how to communicate without learning too much information about you.
 MIN_PY_VERSION = 6
 if sys.version_info[0] == 2 or sys.version_info[1] < MIN_PY_VERSION:
     sys.stderr.write('Error, Onionr requires Python 3.%s+' % (MIN_PY_VERSION,))
@@ -40,10 +44,6 @@ try:
 except ImportError:
     raise Exception("You need the PySocks module (for use with socks5 proxy to use Tor)")
 
-ONIONR_TAGLINE = 'Private P2P Communication - GPLv3 - https://Onionr.net'
-ONIONR_VERSION = '0.0.0' # for debugging and stuff
-ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION)
-API_VERSION = '0' # increments of 1; only change when something fundamental about how the API works changes. This way other nodes know how to communicate without learning too much information about you.
 
 class Onionr:
     def __init__(self):
@@ -72,7 +72,7 @@ class Onionr:
         data_exists = Onionr.setupConfig(self.dataDir, self)
 
         if netcontroller.torBinary() is None:
-            logger.error('Tor is not installed')
+            logger.error('Tor is not installed', terminal=True)
             sys.exit(1)
 
         # If block data folder does not exist
@@ -101,7 +101,6 @@ class Onionr:
         self.onionrCore = core.Core()
         self.onionrCore.onionrInst = self
         #self.deleteRunFiles()
-        self.onionrUtils = onionrutils.OnionrUtils(self.onionrCore)
 
         self.clientAPIInst = '' # Client http api instance
         self.publicAPIInst = '' # Public http api instance
diff --git a/onionr/onionrblockapi.py b/onionr/onionrblockapi.py
index df17aa40..cc823a1f 100755
--- a/onionr/onionrblockapi.py
+++ b/onionr/onionrblockapi.py
@@ -21,6 +21,7 @@
 import core as onionrcore, logger, config, onionrexceptions, nacl.exceptions
 import json, os, sys, datetime, base64, onionrstorage
 from onionrusers import onionrusers
+from onionrutils import stringvalidators
 
 class Block:
     blockCacheOrder = list() # NEVER write your own code that writes to this!
@@ -441,7 +442,7 @@ class Block:
         '''
 
         try:
-            if (not self.isSigned()) or (not self.getCore()._utils.validatePubKey(signer)):
+            if (not self.isSigned()) or (not stringvalidators.validate_pub_key(signer)):
                 return False
 
             return bool(self.getCore()._crypto.edVerify(self.getSignedData(), signer, self.getSignature(), encodedData = encodedData))
diff --git a/onionr/onionrcommands/keyadders.py b/onionr/onionrcommands/keyadders.py
index 363dfe28..b9401d60 100755
--- a/onionr/onionrcommands/keyadders.py
+++ b/onionr/onionrcommands/keyadders.py
@@ -25,7 +25,7 @@ def add_peer(o_inst):
     except IndexError:
         pass
     else:
-        if o_inst.onionrUtils.hasKey(newPeer):
+        if newPeer in o_inst.onionrCore.listPeers():
             logger.info('We already have that key', terminal=True)
             return
         logger.info("Adding peer: " + logger.colors.underline + newPeer, terminal=True)
diff --git a/onionr/onionrcommands/onionrstatistics.py b/onionr/onionrcommands/onionrstatistics.py
index 2f50c6e7..83d62baa 100755
--- a/onionr/onionrcommands/onionrstatistics.py
+++ b/onionr/onionrcommands/onionrstatistics.py
@@ -21,7 +21,7 @@ import os, uuid, time
 import logger, onionrutils
 from onionrblockapi import Block
 import onionr
-from onionrutils import checkcommunicator
+from onionrutils import checkcommunicator, mnemonickeys
 
 def show_stats(o_inst):
     try:
@@ -87,7 +87,7 @@ def show_details(o_inst):
         'Node Address' : o_inst.get_hostname(),
         'Web Password' : o_inst.getWebPassword(),
         'Public Key' : o_inst.onionrCore._crypto.pubKey,
-        'Human-readable Public Key' : o_inst.onionrCore._utils.getHumanReadableID()
+        'Human-readable Public Key' : mnemonickeys.get_human_readable_ID(o_inst.onionrCore)
     }
 
     for detail in details:
diff --git a/onionr/onionrcommands/pubkeymanager.py b/onionr/onionrcommands/pubkeymanager.py
index e8f29b2a..a6b067bc 100755
--- a/onionr/onionrcommands/pubkeymanager.py
+++ b/onionr/onionrcommands/pubkeymanager.py
@@ -20,6 +20,7 @@
 
 import sys, getpass
 import logger, onionrexceptions
+from onionrutils import stringvalidators
 from onionrusers import onionrusers, contactmanager
 import unpaddedbase32
 def add_ID(o_inst):
@@ -55,7 +56,7 @@ def change_ID(o_inst):
     except IndexError:
         logger.warn('Specify pubkey to use', terminal=True)
     else:
-        if o_inst.onionrUtils.validatePubKey(key):
+        if stringvalidators.validate_pub_key(key):
             if key in o_inst.onionrCore._crypto.keyManager.getPubkeyList():
                 o_inst.onionrCore.config.set('general.public_key', key)
                 o_inst.onionrCore.config.save()
@@ -82,7 +83,7 @@ def friend_command(o_inst):
         elif action in ('add', 'remove'):
             try:
                 friend = sys.argv[3]
-                if not o_inst.onionrUtils.validatePubKey(friend):
+                if not stringvalidators.validate_pub_key(friend):
                     raise onionrexceptions.InvalidPubkey('Public key is invalid')
                 if friend not in o_inst.onionrCore.listPeers():
                     raise onionrexceptions.KeyNotKnown
diff --git a/onionr/onionrcrypto.py b/onionr/onionrcrypto.py
index 3c77b3ce..c6dfe300 100755
--- a/onionr/onionrcrypto.py
+++ b/onionr/onionrcrypto.py
@@ -21,7 +21,8 @@ import os, binascii, base64, hashlib, time, sys, hmac, secrets
 import nacl.signing, nacl.encoding, nacl.public, nacl.hash, nacl.pwhash, nacl.utils, nacl.secret
 import unpaddedbase32
 import logger, onionrproofs
-import onionrexceptions, keymanager, core
+from onionrutils import stringvalidators
+import onionrexceptions, keymanager, core, onionrutils
 import config
 config.reload()
 
@@ -38,8 +39,8 @@ class OnionrCrypto:
 
         # Load our own pub/priv Ed25519 keys, gen & save them if they don't exist
         if os.path.exists(self._keyFile):
-            if len(config.get('general.public_key', '')) > 0:
-                self.pubKey = config.get('general.public_key')
+            if len(self._core.config.get('general.public_key', '')) > 0:
+                self.pubKey = self._core.config.get('general.public_key')
             else:
                 self.pubKey = self.keyManager.getPubkeyList()[0]
             self.privKey = self.keyManager.getPrivkey(self.pubKey)
@@ -94,10 +95,10 @@ class OnionrCrypto:
 
     def pubKeyEncrypt(self, data, pubkey, encodedData=False):
         '''Encrypt to a public key (Curve25519, taken from base32 Ed25519 pubkey)'''
-        pubkey = unpaddedbase32.repad(self._core._utils.strToBytes(pubkey))
+        pubkey = unpaddedbase32.repad(onionrutils.str_to_bytes(pubkey))
         retVal = ''
         box = None
-        data = self._core._utils.strToBytes(data)
+        data = onionrutils.str_to_bytes(data)
         
         pubkey = nacl.signing.VerifyKey(pubkey, encoder=nacl.encoding.Base32Encoder()).to_curve25519_public_key()
 
@@ -122,7 +123,7 @@ class OnionrCrypto:
             privkey = self.privKey
         ownKey = nacl.signing.SigningKey(seed=privkey, encoder=nacl.encoding.Base32Encoder()).to_curve25519_private_key()
 
-        if self._core._utils.validatePubKey(privkey):
+        if stringvalidators.validate_pub_key(privkey):
             privkey = nacl.signing.SigningKey(seed=privkey, encoder=nacl.encoding.Base32Encoder()).to_curve25519_private_key()
             anonBox = nacl.public.SealedBox(privkey)
         else:
@@ -181,7 +182,7 @@ class OnionrCrypto:
     def generateDeterministic(self, passphrase, bypassCheck=False):
         '''Generate a Ed25519 public key pair from a password'''
         passStrength = self.deterministicRequirement
-        passphrase = self._core._utils.strToBytes(passphrase) # Convert to bytes if not already
+        passphrase = onionrutils.str_to_bytes(passphrase) # Convert to bytes if not already
         # Validate passphrase length
         if not bypassCheck:
             if len(passphrase) < passStrength:
@@ -201,7 +202,7 @@ class OnionrCrypto:
         if pubkey == '':
             pubkey = self.pubKey
         prev = ''
-        pubkey = self._core._utils.strToBytes(pubkey)
+        pubkey = onionrutils.str_to_bytes(pubkey)
         for i in range(self.HASH_ID_ROUNDS):
             try:
                 prev = prev.encode()
@@ -250,8 +251,8 @@ class OnionrCrypto:
         
         difficulty = onionrproofs.getDifficultyForNewBlock(blockContent, ourBlock=False, coreInst=self._core)
         
-        if difficulty < int(config.get('general.minimum_block_pow')):
-            difficulty = int(config.get('general.minimum_block_pow'))
+        if difficulty < int(self._core.config.get('general.minimum_block_pow')):
+            difficulty = int(self._core.config.get('general.minimum_block_pow'))
         mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode()
         puzzle = mainHash[:difficulty]
 
diff --git a/onionr/onionrproofs.py b/onionr/onionrproofs.py
index 5ff32964..7b8aa91a 100755
--- a/onionr/onionrproofs.py
+++ b/onionr/onionrproofs.py
@@ -18,7 +18,8 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 import multiprocessing, nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, sys, json
-import core, onionrutils, config, logger, onionrblockapi
+import core, config, logger, onionrblockapi
+from onionrutils import bytesconverter
 
 config.reload()
 
@@ -31,8 +32,6 @@ def getDifficultyModifier(coreOrUtilsInst=None):
     retData = 0
     if isinstance(classInst, core.Core):
         useFunc = classInst._utils.storageCounter.getPercent
-    elif isinstance(classInst, onionrutils.OnionrUtils):
-        useFunc = classInst.storageCounter.getPercent
     else:
         useFunc = core.Core()._utils.storageCounter.getPercent
 
@@ -56,7 +55,7 @@ def getDifficultyForNewBlock(data, ourBlock=True, coreInst=None):
     if isinstance(data, onionrblockapi.Block):
         dataSize = len(data.getRaw().encode('utf-8'))
     else:
-        dataSize = len(onionrutils.OnionrUtils.strToBytes(data))
+        dataSize = len(bytesconverter.str_to_bytes(data))
 
     if ourBlock:
         minDifficulty = config.get('general.minimum_send_pow', 4)
diff --git a/onionr/onionrservices/bootstrapservice.py b/onionr/onionrservices/bootstrapservice.py
index e06300e6..777801dd 100755
--- a/onionr/onionrservices/bootstrapservice.py
+++ b/onionr/onionrservices/bootstrapservice.py
@@ -33,7 +33,7 @@ def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=300):
     if core_inst is None:
         core_inst = core.Core()
     
-    if not core_inst._utils.validatePubKey(peer):
+    if not stringvalidators.validate_pub_key(peer):
         raise ValueError('Peer must be valid base32 ed25519 public key')
 
     bootstrap_port = getOpenPort()
diff --git a/onionr/onionrservices/connectionserver.py b/onionr/onionrservices/connectionserver.py
index e676e496..3c4238a7 100755
--- a/onionr/onionrservices/connectionserver.py
+++ b/onionr/onionrservices/connectionserver.py
@@ -24,6 +24,7 @@ import core, logger, httpapi
 import onionrexceptions
 from netcontroller import getOpenPort
 import api
+from onionrutils import stringvalidators
 from . import httpheaders
 
 class ConnectionServer:
@@ -33,7 +34,7 @@ class ConnectionServer:
         else:
             self.core_inst = core_inst
 
-        if not core_inst._utils.validatePubKey(peer):
+        if not stringvalidators.validate_pub_key(peer):
             raise ValueError('Peer must be valid base32 ed25519 public key')
         
         socks = core_inst.config.get('tor.socksport') # Load config for Tor socks port for proxy
diff --git a/onionr/onionrstorage/__init__.py b/onionr/onionrstorage/__init__.py
index b859662d..2af2661b 100755
--- a/onionr/onionrstorage/__init__.py
+++ b/onionr/onionrstorage/__init__.py
@@ -18,6 +18,7 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 import core, sys, sqlite3, os, dbcreator, onionrexceptions
+from onionrutils import bytesconverter
 
 DB_ENTRY_SIZE_LIMIT = 10000 # Will be a config option
 
@@ -82,7 +83,7 @@ def getData(coreInst, bHash):
     assert isinstance(coreInst, core.Core)
     assert coreInst._utils.validateHash(bHash)
 
-    bHash = coreInst._utils.bytesToStr(bHash)
+    bHash = bytesconverter.bytes_to_str(bHash)
 
     # First check DB for data entry by hash
     # if no entry, check disk
diff --git a/onionr/onionrusers/contactmanager.py b/onionr/onionrusers/contactmanager.py
index 098cba65..1cac6953 100755
--- a/onionr/onionrusers/contactmanager.py
+++ b/onionr/onionrusers/contactmanager.py
@@ -20,10 +20,11 @@
 import os, json, onionrexceptions
 import unpaddedbase32
 from onionrusers import onionrusers
+from onionrutils import bytesconverter
 
 class ContactManager(onionrusers.OnionrUser):
     def __init__(self, coreInst, publicKey, saveUser=False, recordExpireSeconds=5):
-        publicKey = unpaddedbase32.repad(coreInst._utils.strToBytes(publicKey)).decode()
+        publicKey = unpaddedbase32.repad(bytesconverter.str_to_bytes(publicKey)).decode()
         super(ContactManager, self).__init__(coreInst, publicKey, saveUser=saveUser)
         self.dataDir = coreInst.dataDir + '/contacts/'
         self.dataFile = '%s/contacts/%s.json' % (coreInst.dataDir, publicKey)
diff --git a/onionr/onionrusers/onionrusers.py b/onionr/onionrusers/onionrusers.py
index 51977b39..5bb2eac8 100755
--- a/onionr/onionrusers/onionrusers.py
+++ b/onionr/onionrusers/onionrusers.py
@@ -17,7 +17,9 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import onionrblockapi, logger, onionrexceptions, json, sqlite3, time
+import logger, onionrexceptions, json, sqlite3, time
+from onionrutils import stringvalidators, bytesconverter
+
 import unpaddedbase32
 import nacl.exceptions
 
@@ -56,7 +58,7 @@ class OnionrUser:
             Takes an instance of onionr core, a base32 encoded ed25519 public key, and a bool saveUser
             saveUser determines if we should add a user to our peer database or not.
         '''
-        publicKey = unpaddedbase32.repad(coreInst._utils.strToBytes(publicKey)).decode()
+        publicKey = unpaddedbase32.repad(bytesconverter.str_to_bytes(publicKey)).decode()
 
         self.trust = 0
         self._core = coreInst
@@ -103,7 +105,7 @@ class OnionrUser:
         deleteExpiredKeys(self._core)
         retData = ''
         forwardKey = self._getLatestForwardKey()
-        if self._core._utils.validatePubKey(forwardKey[0]):
+        if stringvalidators.validate_pub_key(forwardKey[0]):
             retData = self._core._crypto.pubKeyEncrypt(data, forwardKey[0], encodedData=True)
         else:
             raise onionrexceptions.InvalidPubkey("No valid forward secrecy key available for this user")
@@ -190,8 +192,8 @@ class OnionrUser:
         return list(keyList)
 
     def addForwardKey(self, newKey, expire=DEFAULT_KEY_EXPIRE):
-        newKey = self._core._utils.bytesToStr(unpaddedbase32.repad(self._core._utils.strToBytes(newKey)))
-        if not self._core._utils.validatePubKey(newKey):
+        newKey = self._core._utils.bytesToStr(unpaddedbase32.repad(bytesconverter.str_to_bytes(newKey)))
+        if not stringvalidators.validate_pub_key(newKey):
             # Do not add if something went wrong with the key
             raise onionrexceptions.InvalidPubkey(newKey)
 
diff --git a/onionr/onionrutils/__init__.py b/onionr/onionrutils/__init__.py
old mode 100755
new mode 100644
index 23907a00..e430a7b1
--- a/onionr/onionrutils/__init__.py
+++ b/onionr/onionrutils/__init__.py
@@ -22,13 +22,11 @@ import sys, os, sqlite3, binascii, time, base64, json, glob, shutil, math, re, u
 import requests
 import nacl.signing, nacl.encoding
 import unpaddedbase32
-from onionrblockapi import Block
 import onionrexceptions, config, logger
 import onionrevents
 import storagecounter
 from etc import pgpwords, onionrvalues
-from onionrusers import onionrusers 
-from . import localcommand, blockmetadata, validatemetadata, basicrequests
+from . import localcommand, blockmetadata, basicrequests, validatemetadata
 from . import stringvalidators
 
 config.reload()
@@ -45,39 +43,6 @@ class OnionrUtils:
         self.storageCounter = storagecounter.StorageCounter(self._core) # used to keep track of how much data onionr is using on disk
         return
 
-    def getRoundedEpoch(self, roundS=60):
-        '''
-            Returns the epoch, rounded down to given seconds (Default 60)
-        '''
-        epoch = self.getEpoch()
-        return epoch - (epoch % roundS)
-
-    def getHumanReadableID(self, pub=''):
-        '''gets a human readable ID from a public key'''
-        if pub == '':
-            pub = self._core._crypto.pubKey
-        pub = base64.b16encode(base64.b32decode(pub)).decode()
-        return ' '.join(pgpwords.wordify(pub))
-    
-    def convertHumanReadableID(self, pub):
-        '''Convert a human readable pubkey id to base32'''
-        pub = pub.lower()
-        return self.bytesToStr(base64.b32encode(binascii.unhexlify(pgpwords.hexify(pub.strip()))))
-
-    def getBlockMetadataFromData(self, blockData):
-        '''
-            accepts block contents as string, returns a tuple of 
-            metadata, meta (meta being internal metadata, which will be 
-            returned as an encrypted base64 string if it is encrypted, dict if not).
-        '''
-        return blockmetadata.get_block_metadata_from_data(self, blockData)
-
-    def processBlockMetadata(self, blockHash):
-        '''
-            Read metadata from a block and cache it to the block database
-        '''
-        return blockmetadata.process_block_metadata(self, blockHash)
-
     def escapeAnsi(self, line):
         '''
             Remove ANSI escape codes from a string with regex
@@ -88,46 +53,12 @@ class OnionrUtils:
         ansi_escape = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -/]*[@-~]')
         return ansi_escape.sub('', line)
 
-    def hasBlock(self, hash):
-        '''
-            Check for new block in the list
-        '''
-        conn = sqlite3.connect(self._core.blockDB)
-        c = conn.cursor()
-        if not self.validateHash(hash):
-            raise Exception("Invalid hash")
-        for result in c.execute("SELECT COUNT() FROM hashes WHERE hash = ?", (hash,)):
-            if result[0] >= 1:
-                conn.commit()
-                conn.close()
-                return True
-            else:
-                conn.commit()
-                conn.close()
-                return False
-
-    def hasKey(self, key):
-        '''
-            Check for key in list of public keys
-        '''
-        return key in self._core.listPeers()
-
     def validateHash(self, data, length=64):
         '''
             Validate if a string is a valid hash hex digest (does not compare, just checks length and charset)
         '''
         return stringvalidators.validate_hash(self, data, length)
 
-    def validateMetadata(self, metadata, blockData):
-        '''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string'''
-        return validatemetadata.validate_metadata(self, metadata, blockData)
-
-    def validatePubKey(self, key):
-        '''
-            Validate if a string is a valid base32 encoded Ed25519 key
-        '''
-        return stringvalidators.validate_pub_key(self, key)
-
     def getEpoch(self):
         '''returns epoch'''
         return math.floor(time.time())
@@ -144,21 +75,6 @@ class OnionrUtils:
         '''
         return basicrequests.do_get_request(self, url, port, proxyType, ignoreAPI, returnHeaders)
 
-    @staticmethod
-    def strToBytes(data):
-        try:
-            data = data.encode()
-        except AttributeError:
-            pass
-        return data
-    @staticmethod
-    def bytesToStr(data):
-        try:
-            data = data.decode()
-        except AttributeError:
-            pass
-        return data
-
 def size(path='.'):
     '''
         Returns the size of a folder's contents in bytes
@@ -183,4 +99,22 @@ def humanSize(num, suffix='B'):
         if abs(num) < 1024.0:
             return "%.1f %s%s" % (num, unit, suffix)
         num /= 1024.0
-    return "%.1f %s%s" % (num, 'Yi', suffix)
\ No newline at end of file
+    return "%.1f %s%s" % (num, 'Yi', suffix)
+
+def has_block(core_inst, hash):
+    '''
+        Check for new block in the list
+    '''
+    conn = sqlite3.connect(core_inst.blockDB)
+    c = conn.cursor()
+    if not stringvalidators.validate_hash(hash):
+        raise Exception("Invalid hash")
+    for result in c.execute("SELECT COUNT() FROM hashes WHERE hash = ?", (hash,)):
+        if result[0] >= 1:
+            conn.commit()
+            conn.close()
+            return True
+        else:
+            conn.commit()
+            conn.close()
+            return False
\ No newline at end of file
diff --git a/onionr/onionrutils/basicrequests.py b/onionr/onionrutils/basicrequests.py
index f9432928..e889b887 100644
--- a/onionr/onionrutils/basicrequests.py
+++ b/onionr/onionrutils/basicrequests.py
@@ -1,6 +1,5 @@
 import requests
 import logger, onionrexceptions
-from onionr import API_VERSION
 def do_post_request(utils_inst, url, data={}, port=0, proxyType='tor'):
     '''
     Do a POST request through a local tor or i2p instance
@@ -29,6 +28,7 @@ def do_get_request(utils_inst, url, port=0, proxyType='tor', ignoreAPI=False, re
     '''
     Do a get request through a local tor or i2p instance
     '''
+    API_VERSION = utils_inst._core.onionrInst.API_VERSION
     retData = False
     if proxyType == 'tor':
         if port == 0:
diff --git a/onionr/onionrutils/blockmetadata.py b/onionr/onionrutils/blockmetadata.py
index 0515ffc7..ace85190 100644
--- a/onionr/onionrutils/blockmetadata.py
+++ b/onionr/onionrutils/blockmetadata.py
@@ -2,8 +2,9 @@ import json
 import logger, onionrevents
 from onionrusers import onionrusers
 from etc import onionrvalues
-from onionrblockapi import Block
-def get_block_metadata_from_data(utils_inst, blockData):
+import onionrblockapi
+from . import epoch
+def get_block_metadata_from_data(blockData):
     '''
         accepts block contents as string, returns a tuple of 
         metadata, meta (meta being internal metadata, which will be 
@@ -36,8 +37,8 @@ def process_block_metadata(utils_inst, blockHash):
     '''
         Read metadata from a block and cache it to the block database
     '''
-    curTime = utils_inst.getRoundedEpoch(roundS=60)
-    myBlock = Block(blockHash, utils_inst._core)
+    curTime = epoch.get_rounded_epoch(roundS=60)
+    myBlock = onionrblockapi.Block(blockHash, utils_inst._core)
     if myBlock.isEncrypted:
         myBlock.decrypt()
     if (myBlock.isEncrypted and myBlock.decrypted) or (not myBlock.isEncrypted):
diff --git a/onionr/onionrutils/bytesconverter.py b/onionr/onionrutils/bytesconverter.py
new file mode 100644
index 00000000..cf69a91d
--- /dev/null
+++ b/onionr/onionrutils/bytesconverter.py
@@ -0,0 +1,13 @@
+def str_to_bytes(data):
+    try:
+        data = data.encode()
+    except AttributeError:
+        pass
+    return data
+
+def bytes_to_str(data):
+    try:
+        data = data.decode()
+    except AttributeError:
+        pass
+    return data
\ No newline at end of file
diff --git a/onionr/onionrutils/epoch.py b/onionr/onionrutils/epoch.py
new file mode 100644
index 00000000..1ee8ae25
--- /dev/null
+++ b/onionr/onionrutils/epoch.py
@@ -0,0 +1,11 @@
+import math, time
+def get_rounded_epoch(roundS=60):
+    '''
+        Returns the epoch, rounded down to given seconds (Default 60)
+    '''
+    epoch = get_epoch()
+    return epoch - (epoch % roundS)
+
+def get_epoch(self):
+    '''returns epoch'''
+    return math.floor(time.time())
\ No newline at end of file
diff --git a/onionr/onionrutils/importnewblocks.py b/onionr/onionrutils/importnewblocks.py
index a2c1e518..bfd985a2 100644
--- a/onionr/onionrutils/importnewblocks.py
+++ b/onionr/onionrutils/importnewblocks.py
@@ -1,5 +1,6 @@
 import glob
 import logger, core
+from onionrutils import blockmetadata
 def import_new_blocks(core_inst=None, scanDir=''):
     '''
         This function is intended to scan for new blocks ON THE DISK and import them
@@ -21,7 +22,7 @@ def import_new_blocks(core_inst=None, scanDir=''):
                 if core_inst._crypto.sha3Hash(newBlock.read()) == block.replace('.dat', ''):
                     core_inst.addToBlockDB(block.replace('.dat', ''), dataSaved=True)
                     logger.info('Imported block %s.' % block)
-                    core_inst._utils.processBlockMetadata(block)
+                    blockmetadata.process_block_metadata(block)
                 else:
                     logger.warn('Failed to verify hash for %s' % block)
     if not exist:
diff --git a/onionr/onionrutils/mnemonickeys.py b/onionr/onionrutils/mnemonickeys.py
new file mode 100644
index 00000000..30c7b1c4
--- /dev/null
+++ b/onionr/onionrutils/mnemonickeys.py
@@ -0,0 +1,8 @@
+import base64
+from etc import pgpwords
+def get_human_readable_ID(core_inst, pub=''):
+    '''gets a human readable ID from a public key'''
+    if pub == '':
+        pub = core_inst._crypto.pubKey
+    pub = base64.b16encode(base64.b32decode(pub)).decode()
+    return ' '.join(pgpwords.wordify(pub))
diff --git a/onionr/onionrutils/stringvalidators.py b/onionr/onionrutils/stringvalidators.py
index 49f2c33a..cdfd2338 100644
--- a/onionr/onionrutils/stringvalidators.py
+++ b/onionr/onionrutils/stringvalidators.py
@@ -1,4 +1,4 @@
-import base64, string
+import base64, string, onionrutils
 import unpaddedbase32, nacl.signing, nacl.encoding
 def validate_hash(utils_inst, data, length=64):
     '''
@@ -18,14 +18,14 @@ def validate_hash(utils_inst, data, length=64):
 
     return retVal
 
-def validate_pub_key(utils_inst, key):
+def validate_pub_key(key):
     '''
         Validate if a string is a valid base32 encoded Ed25519 key
     '''
     if type(key) is type(None):
         return False
     # Accept keys that have no = padding
-    key = unpaddedbase32.repad(utils_inst.strToBytes(key))
+    key = unpaddedbase32.repad(onionrutils.str_to_bytes(key))
 
     retVal = False
     try:
diff --git a/onionr/onionrutils/validatemetadata.py b/onionr/onionrutils/validatemetadata.py
index 8feb9859..aebd45e8 100644
--- a/onionr/onionrutils/validatemetadata.py
+++ b/onionr/onionrutils/validatemetadata.py
@@ -2,7 +2,7 @@ import json
 import logger, onionrexceptions
 from etc import onionrvalues
 from onionrutils import stringvalidators
-def validate_metadata(utils_inst, metadata, blockData):
+def validate_metadata(core_inst, metadata, blockData):
     '''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string'''
     # TODO, make this check sane sizes
     retData = False
@@ -16,11 +16,11 @@ def validate_metadata(utils_inst, metadata, blockData):
             pass
 
     # Validate metadata dict for invalid keys to sizes that are too large
-    maxAge = utils_inst._core.config.get("general.max_block_age", onionrvalues.OnionrValues().default_expire)
+    maxAge = core_inst.config.get("general.max_block_age", onionrvalues.OnionrValues().default_expire)
     if type(metadata) is dict:
         for i in metadata:
             try:
-                utils_inst._core.requirements.blockMetadataLengths[i]
+                core_inst.requirements.blockMetadataLengths[i]
             except KeyError:
                 logger.warn('Block has invalid metadata key ' + i)
                 break
@@ -30,25 +30,25 @@ def validate_metadata(utils_inst, metadata, blockData):
                     testData = len(testData)
                 except (TypeError, AttributeError) as e:
                     testData = len(str(testData))
-                if utils_inst._core.requirements.blockMetadataLengths[i] < testData:
+                if core_inst.requirements.blockMetadataLengths[i] < testData:
                     logger.warn('Block metadata key ' + i + ' exceeded maximum size')
                     break
             if i == 'time':
                 if not stringvalidators.is_integer_string(metadata[i]):
                     logger.warn('Block metadata time stamp is not integer string or int')
                     break
-                isFuture = (metadata[i] - utils_inst.getEpoch())
+                isFuture = (metadata[i] - core_inst.getEpoch())
                 if isFuture > maxClockDifference:
                     logger.warn('Block timestamp is skewed to the future over the max %s: %s' (maxClockDifference, isFuture))
                     break
-                if (utils_inst.getEpoch() - metadata[i]) > maxAge:
+                if (core_inst.getEpoch() - metadata[i]) > maxAge:
                     logger.warn('Block is outdated: %s' % (metadata[i],))
                     break
             elif i == 'expire':
                 try:
-                    assert int(metadata[i]) > utils_inst.getEpoch()
+                    assert int(metadata[i]) > core_inst.getEpoch()
                 except AssertionError:
-                    logger.warn('Block is expired: %s less than %s' % (metadata[i], utils_inst.getEpoch()))
+                    logger.warn('Block is expired: %s less than %s' % (metadata[i], core_inst.getEpoch()))
                     break
             elif i == 'encryptType':
                 try:
@@ -59,9 +59,9 @@ def validate_metadata(utils_inst, metadata, blockData):
         else:
             # if metadata loop gets no errors, it does not break, therefore metadata is valid
             # make sure we do not have another block with the same data content (prevent data duplication and replay attacks)
-            nonce = utils_inst._core._utils.bytesToStr(utils_inst._core._crypto.sha3Hash(blockData))
+            nonce = core_inst._utils.bytesToStr(core_inst._crypto.sha3Hash(blockData))
             try:
-                with open(utils_inst._core.dataNonceFile, 'r') as nonceFile:
+                with open(core_inst.dataNonceFile, 'r') as nonceFile:
                     if nonce in nonceFile.read():
                         retData = False # we've seen that nonce before, so we can't pass metadata
                         raise onionrexceptions.DataExists
diff --git a/onionr/static-data/default-plugins/encrypt/main.py b/onionr/static-data/default-plugins/encrypt/main.py
index 8c874c99..5e5590ad 100755
--- a/onionr/static-data/default-plugins/encrypt/main.py
+++ b/onionr/static-data/default-plugins/encrypt/main.py
@@ -21,6 +21,7 @@
 # Imports some useful libraries
 import logger, config, threading, time, datetime, sys, json
 from onionrblockapi import Block
+from onionrutils import stringvalidators
 import onionrexceptions, onionrusers
 import locale
 locale.setlocale(locale.LC_ALL, '')
@@ -43,7 +44,7 @@ class PlainEncryption:
             pass
 
         try:
-            if not self.api.get_core()._utils.validatePubKey(sys.argv[2]):
+            if not stringvalidators.validate_pub_key(sys.argv[2]):
                 raise onionrexceptions.InvalidPubkey
         except (ValueError, IndexError) as e:
             logger.error("Peer public key not specified", terminal=True)
diff --git a/onionr/static-data/default-plugins/esoteric/main.py b/onionr/static-data/default-plugins/esoteric/main.py
index 1f4cdd32..0c6ae8fa 100755
--- a/onionr/static-data/default-plugins/esoteric/main.py
+++ b/onionr/static-data/default-plugins/esoteric/main.py
@@ -23,6 +23,7 @@ import locale, sys, os, threading, json
 locale.setlocale(locale.LC_ALL, '')
 import onionrservices, logger
 from onionrservices import bootstrapservice
+from onionrutils import stringvalidators
 
 plugin_name = 'esoteric'
 PLUGIN_VERSION = '0.0.0'
@@ -66,7 +67,7 @@ class Esoteric:
     def create(self):
         try:
             peer = sys.argv[2]
-            if not self.myCore._utils.validatePubKey(peer):
+            if not stringvalidators.validate_pub_key(peer):
                 exit_with_error('Invalid public key specified')
         except IndexError:
             exit_with_error('You must specify a peer public key')
diff --git a/onionr/static-data/default-plugins/metadataprocessor/main.py b/onionr/static-data/default-plugins/metadataprocessor/main.py
index 333c68aa..4c730783 100755
--- a/onionr/static-data/default-plugins/metadataprocessor/main.py
+++ b/onionr/static-data/default-plugins/metadataprocessor/main.py
@@ -23,6 +23,7 @@ import logger, config
 import os, sys, json, time, random, shutil, base64, getpass, datetime, re
 from onionrblockapi import Block
 import onionrusers, onionrexceptions
+from onionrutils import stringvalidators
 
 plugin_name = 'metadataprocessor'
 
@@ -36,7 +37,7 @@ def _processForwardKey(api, myBlock):
     key = myBlock.getMetadata('newFSKey')
 
     # We don't need to validate here probably, but it helps
-    if api.get_utils().validatePubKey(key):
+    if stringvalidators.validate_pub_key(key):
         peer.addForwardKey(key)
     else:
         raise onionrexceptions.InvalidPubkey("%s is not a valid pubkey key" % (key,))
diff --git a/onionr/static-data/default-plugins/pluginmanager/main.py b/onionr/static-data/default-plugins/pluginmanager/main.py
index 2f261bbf..bb21d35c 100755
--- a/onionr/static-data/default-plugins/pluginmanager/main.py
+++ b/onionr/static-data/default-plugins/pluginmanager/main.py
@@ -22,7 +22,7 @@
 import logger, config
 import os, sys, json, time, random, shutil, base64, getpass, datetime, re
 from onionrblockapi import Block
-from onionrutils import importnewblocks
+from onionrutils import importnewblocks, stringvalidators, 
 
 plugin_name = 'pluginmanager'
 
@@ -399,13 +399,13 @@ def commandInstallPlugin():
 
         valid_hash = pluginapi.get_utils().validateHash(pkobh)
         real_block = False
-        valid_key = pluginapi.get_utils().validatePubKey(pkobh)
+        valid_key = stringvalidators.validate_pub_key(pkobh)
         real_key = False
 
         if valid_hash:
             real_block = Block.exists(pkobh)
         elif valid_key:
-            real_key = pluginapi.get_utils().hasKey(pkobh)
+            real_key = pkobh in pluginapi.get_core().listPeers()
 
         blockhash = None
 
@@ -493,7 +493,7 @@ def commandAddRepository():
                     pluginslist = dict()
 
                     for pluginname, distributor in blockContent['plugins']:
-                        if pluginapi.get_utils().validatePubKey(distributor):
+                        if stringvalidators.validate_pub_key(distributor):
                             pluginslist[pluginname] = distributor
 
                     logger.debug('Found %s records in repository.' % len(pluginslist), terminal=True)
diff --git a/onionr/static-data/default-plugins/pms/main.py b/onionr/static-data/default-plugins/pms/main.py
index e594c368..532320de 100755
--- a/onionr/static-data/default-plugins/pms/main.py
+++ b/onionr/static-data/default-plugins/pms/main.py
@@ -23,6 +23,7 @@ import logger, config, threading, time, datetime
 from onionrblockapi import Block
 import onionrexceptions
 from onionrusers import onionrusers
+from onionrutils import stringvalidators
 import locale, sys, os, json
 
 locale.setlocale(locale.LC_ALL, '')
@@ -217,7 +218,7 @@ class OnionrMail:
                     recip = logger.readline('Enter peer address, or -q to stop:').strip()
                     if recip in ('-q', 'q'):
                         raise EOFError
-                    if not self.myCore._utils.validatePubKey(recip):
+                    if not stringvalidators.validate_pub_key(recip):
                         raise onionrexceptions.InvalidPubkey('Must be a valid ed25519 base32 encoded public key')
                 except onionrexceptions.InvalidPubkey:
                     logger.warn('Invalid public key', terminal=True)
diff --git a/onionr/subprocesspow.py b/onionr/subprocesspow.py
index 52a90c64..1ca2f9b1 100755
--- a/onionr/subprocesspow.py
+++ b/onionr/subprocesspow.py
@@ -23,6 +23,7 @@ import subprocess, os
 import multiprocessing, threading, time, json
 from multiprocessing import Pipe, Process
 import core, onionrblockapi, config, onionrutils, logger, onionrproofs
+from onionrutils import bytesconverter
 
 class SubprocessPOW:
     def __init__(self, data, metadata, core_inst=None, subproc_count=None):
@@ -51,7 +52,7 @@ class SubprocessPOW:
         # dump dict to measure bytes of json metadata. Cannot reuse later because the pow token must be added
         json_metadata = json.dumps(metadata).encode()
 
-        self.data = onionrutils.OnionrUtils.strToBytes(data)
+        self.data = bytesconverter.str_to_bytes(data)
         # Calculate difficulty. Dumb for now, may use good algorithm in the future.
         self.difficulty = onionrproofs.getDifficultyForNewBlock(bytes(json_metadata + b'\n' + self.data), coreInst=self.core_inst)
         
@@ -111,7 +112,7 @@ class SubprocessPOW:
             payload = json.dumps(metadata).encode() + b'\n' + data
             # Check sha3_256 hash of block, compare to puzzle. Send payload if puzzle finished
             token = mcore._crypto.sha3Hash(payload)
-            token = onionrutils.OnionrUtils.bytesToStr(token) # ensure token is string
+            token = bytesconverter.bytes_to_str(token) # ensure token is string
             if puzzle == token[0:difficulty]:
                 pipe.send(payload)
                 break
diff --git a/onionr/tests/test_highlevelcrypto.py b/onionr/tests/test_highlevelcrypto.py
index 0a675ed7..aab2e198 100755
--- a/onionr/tests/test_highlevelcrypto.py
+++ b/onionr/tests/test_highlevelcrypto.py
@@ -4,6 +4,7 @@ sys.path.append(".")
 import unittest, uuid, hashlib, base64
 import nacl.exceptions
 import nacl.signing, nacl.hash, nacl.encoding
+from onionrutils import stringvalidators, mnemonickeys
 TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
 print("Test directory:", TEST_DIR)
 os.environ["ONIONR_HOME"] = TEST_DIR
@@ -45,19 +46,12 @@ class OnionrCryptoTests(unittest.TestCase):
         self.assertEqual(crypto.sha3Hash(b'test'), normal)
         
     def valid_default_id(self):
-        self.assertTrue(c._utils.validatePubKey(crypto.pubKey))
+        self.assertTrue(stringvalidators.validate_pub_key(crypto.pubKey))
     
     def test_human_readable_length(self):
-        human = c._utils.getHumanReadableID()
+        human = mnemonickeys.get_human_readable_ID(c)
         self.assertTrue(len(human.split(' ')) == 32)
     
-    def test_human_readable_rebuild(self):
-        return # Broken right now
-        # Test if we can get the human readable id, and convert it back to valid base32 key
-        human = c._utils.getHumanReadableID()
-        unHuman = c._utils.convertHumanReadableID(human)
-        nacl.signing.VerifyKey(c._utils.convertHumanReadableID(human), encoder=nacl.encoding.Base32Encoder)
-    
     def test_safe_compare(self):
         self.assertTrue(crypto.safeCompare('test', 'test'))
         self.assertTrue(crypto.safeCompare('test', b'test'))
@@ -130,7 +124,7 @@ class OnionrCryptoTests(unittest.TestCase):
     def test_deterministic(self):
         password = os.urandom(32)
         gen = crypto.generateDeterministic(password)
-        self.assertTrue(c._utils.validatePubKey(gen[0]))
+        self.assertTrue(stringvalidators.validate_pub_key(gen[0]))
         try:
             crypto.generateDeterministic('weakpassword')
         except onionrexceptions.PasswordStrengthError:
@@ -151,6 +145,6 @@ class OnionrCryptoTests(unittest.TestCase):
         gen2 = crypto.generateDeterministic(password)
         self.assertFalse(gen == gen1)
         self.assertTrue(gen1 == gen2)
-        self.assertTrue(c._utils.validatePubKey(gen1[0]))
+        self.assertTrue(stringvalidators.validate_pub_key(gen1[0]))
 
 unittest.main()
\ No newline at end of file

From c7e06205b7f7ab80633484043f66f723dfc63a62 Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Tue, 25 Jun 2019 18:07:35 -0500
Subject: [PATCH 14/29] OnionrUtils fully removed (but not fully bug free) flow
 now uses daemon thread for displaying output

---
 onionr/api.py                                 |  24 ++--
 onionr/communicator.py                        |  12 +-
 onionr/communicatorutils/announcenode.py      |   5 +-
 onionr/communicatorutils/connectnewpeers.py   |   4 +-
 onionr/communicatorutils/cooldownpeer.py      |   7 +-
 onionr/communicatorutils/downloadblocks.py    |   6 +-
 onionr/communicatorutils/housekeeping.py      |   5 +-
 onionr/communicatorutils/lookupblocks.py      |   8 +-
 onionr/communicatorutils/netcheck.py          |   4 +-
 onionr/communicatorutils/uploadblocks.py      |  11 +-
 onionr/core.py                                | 133 +++++++++---------
 onionr/coredb/blockmetadb/add.py              |   6 +-
 onionr/coredb/blockmetadb/expiredblocks.py    |   3 +-
 onionr/coredb/daemonqueue/__init__.py         |   4 +-
 onionr/coredb/keydb/listkeys.py               |   3 +-
 onionr/httpapi/miscpublicapi/getblocks.py     |   7 +-
 onionr/keymanager.py                          |   6 +-
 onionr/onionr.py                              |   2 +-
 onionr/onionrblacklist.py                     |   9 +-
 onionr/onionrblockapi.py                      |   4 +-
 onionr/onionrcommands/banblocks.py            |   3 +-
 onionr/onionrcommands/exportblocks.py         |   3 +-
 onionr/onionrcommands/filecommands.py         |   3 +-
 onionr/onionrcrypto.py                        |  12 +-
 onionr/onionrpeers.py                         |   3 +-
 onionr/onionrpluginapi.py                     |   3 -
 onionr/onionrproofs.py                        |   5 +-
 onionr/onionrservices/__init__.py             |   4 +-
 onionr/onionrservices/bootstrapservice.py     |   4 +-
 onionr/onionrservices/connectionserver.py     |   4 +-
 onionr/onionrstorage/__init__.py              |   6 +-
 onionr/onionrstorage/removeblock.py           |   5 +-
 onionr/onionrstorage/setdata.py               |   2 +-
 onionr/onionrusers/contactmanager.py          |   6 +-
 onionr/onionrusers/onionrusers.py             |  19 ++-
 onionr/onionrutils/__init__.py                | 120 ----------------
 onionr/onionrutils/basicrequests.py           |   8 +-
 onionr/onionrutils/blockmetadata.py           |  41 ++++--
 onionr/onionrutils/epoch.py                   |   2 +-
 onionr/onionrutils/escapeansi.py              |  10 ++
 onionr/onionrutils/stringvalidators.py        |   7 +-
 onionr/onionrutils/validatemetadata.py        |  10 +-
 .../default-plugins/esoteric/main.py          |   8 +-
 .../static-data/default-plugins/flow/main.py  |   7 +-
 .../default-plugins/pluginmanager/main.py     |   8 +-
 .../default-plugins/pms/mailapi.py            |   3 +-
 .../static-data/default-plugins/pms/main.py   |   6 +-
 .../default-plugins/pms/sentboxdb.py          |   3 +-
 onionr/utils/netutils.py                      |   5 +-
 onionr/utils/sizeutils.py                     |  27 ++++
 50 files changed, 280 insertions(+), 330 deletions(-)
 create mode 100644 onionr/onionrutils/escapeansi.py
 create mode 100644 onionr/utils/sizeutils.py

diff --git a/onionr/api.py b/onionr/api.py
index 7d8bf671..dcd912af 100755
--- a/onionr/api.py
+++ b/onionr/api.py
@@ -23,11 +23,12 @@ from gevent import Timeout
 import flask
 from flask import request, Response, abort, send_from_directory
 import core
-import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config, onionrblockapi
+import onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config, onionrblockapi
 import httpapi
 from httpapi import friendsapi, profilesapi, configapi, miscpublicapi
 from onionrservices import httpheaders
 import onionr
+from onionrutils import bytesconverter, stringvalidators, epoch, mnemonickeys
 
 config.reload()
 class FDSafeHandler(WSGIHandler):
@@ -98,7 +99,7 @@ class PublicAPI:
             resp = httpheaders.set_default_onionr_http_headers(resp)
             # Network API version
             resp.headers['X-API'] = onionr.API_VERSION
-            self.lastRequest = clientAPI._core._utils.getRoundedEpoch(roundS=5)
+            self.lastRequest = epoch.get_rounded_epoch(roundS=5)
             return resp
 
         @app.route('/')
@@ -177,9 +178,8 @@ class API:
 
         self.debug = debug
         self._core = onionrInst.onionrCore
-        self.startTime = self._core._utils.getEpoch()
+        self.startTime = epoch.get_epoch()
         self._crypto = onionrcrypto.OnionrCrypto(self._core)
-        self._utils = onionrutils.OnionrUtils(self._core)
         app = flask.Flask(__name__)
         bindPort = int(config.get('client.client.port', 59496))
         self.bindPort = bindPort
@@ -334,7 +334,7 @@ class API:
         @app.route('/getblockbody/<name>')
         def getBlockBodyData(name):
             resp = ''
-            if self._core._utils.validateHash(name):
+            if stringvalidators.validate_hash(name):
                 try:
                     resp = onionrblockapi.Block(name, decrypt=True).bcontent
                 except TypeError:
@@ -346,7 +346,7 @@ class API:
         @app.route('/getblockdata/<name>')
         def getData(name):
             resp = ""
-            if self._core._utils.validateHash(name):
+            if stringvalidators.validate_hash(name):
                 if name in self._core.getBlockList():
                     try:
                         resp = self.getBlockData(name, decrypt=True)
@@ -371,7 +371,7 @@ class API:
         def site(name):
             bHash = name
             resp = 'Not Found'
-            if self._core._utils.validateHash(bHash):
+            if stringvalidators.validate_hash(bHash):
                 try:
                     resp = onionrblockapi.Block(bHash).bcontent
                 except onionrexceptions.NoDataAvailable:
@@ -432,7 +432,7 @@ class API:
 
         @app.route('/getHumanReadable/<name>')
         def getHumanReadable(name):
-            return Response(self._core._utils.getHumanReadableID(name))
+            return Response(mnemonickeys.get_human_readable_ID(name))
 
         @app.route('/insertblock', methods=['POST'])
         def insertBlock():
@@ -497,13 +497,13 @@ class API:
     def getUptime(self):
         while True:
             try:
-                return self._utils.getEpoch() - self.startTime
+                return epoch.get_epoch() - self.startTime
             except (AttributeError, NameError):
                 # Don't error on race condition with startup
                 pass
     
     def getBlockData(self, bHash, decrypt=False, raw=False, headerOnly=False):
-        assert self._core._utils.validateHash(bHash)
+        assert stringvalidators.validate_hash(bHash)
         bl = onionrblockapi.Block(bHash, core=self._core)
         if decrypt:
             bl.decrypt()
@@ -520,8 +520,8 @@ class API:
                         pass
             else:
                 validSig = False
-                signer = onionrutils.bytes_to_str(bl.signer)
-                if bl.isSigned() and onionrutils.stringvalidators.validate_pub_key(signer) and bl.isSigner(signer):
+                signer = bytesconverter.bytes_to_str(bl.signer)
+                if bl.isSigned() and stringvalidators.validate_pub_key(signer) and bl.isSigner(signer):
                     validSig = True                    
                 bl.bheader['validSig'] = validSig
                 bl.bheader['meta'] = ''
diff --git a/onionr/communicator.py b/onionr/communicator.py
index 57917b9a..521f93b6 100755
--- a/onionr/communicator.py
+++ b/onionr/communicator.py
@@ -27,7 +27,7 @@ from communicatorutils import downloadblocks, lookupblocks, lookupadders
 from communicatorutils import servicecreator, connectnewpeers, uploadblocks
 from communicatorutils import daemonqueuehandler, announcenode, deniableinserts
 from communicatorutils import cooldownpeer, housekeeping, netcheck
-from onionrutils import localcommand
+from onionrutils import localcommand, epoch, basicrequests
 from etc import humanreadabletime
 import onionrservices, onionr, onionrproofs
 
@@ -91,7 +91,7 @@ class OnionrCommunicatorDaemon:
         plugins.reload()
 
         # time app started running for info/statistics purposes
-        self.startTime = self._core._utils.getEpoch()
+        self.startTime = epoch.get_epoch()
 
         if developmentMode:
             OnionrCommunicatorTimers(self, self.heartbeat, 30)
@@ -310,9 +310,9 @@ class OnionrCommunicatorDaemon:
         if len(data) > 0:
             url += '&data=' + data
 
-        self._core.setAddressInfo(peer, 'lastConnectAttempt', self._core._utils.getEpoch()) # mark the time we're trying to request this peer
+        self._core.setAddressInfo(peer, 'lastConnectAttempt', epoch.get_epoch()) # mark the time we're trying to request this peer
 
-        retData = self._core._utils.doGetRequest(url, port=self.proxyPort)
+        retData = basicrequests.do_get_request(self._core, url, port=self.proxyPort)
         # if request failed, (error), mark peer offline
         if retData == False:
             try:
@@ -324,7 +324,7 @@ class OnionrCommunicatorDaemon:
             except ValueError:
                 pass
         else:
-            self._core.setAddressInfo(peer, 'lastConnect', self._core._utils.getEpoch())
+            self._core.setAddressInfo(peer, 'lastConnect', epoch.get_epoch())
             self.getPeerProfileInstance(peer).addScore(1)
         return retData # If returnHeaders, returns tuple of data, headers. if not, just data string
 
@@ -341,7 +341,7 @@ class OnionrCommunicatorDaemon:
         return retData
 
     def getUptime(self):
-        return self._core._utils.getEpoch() - self.startTime
+        return epoch.get_epoch() - self.startTime
 
     def heartbeat(self):
         '''Show a heartbeat debug message'''
diff --git a/onionr/communicatorutils/announcenode.py b/onionr/communicatorutils/announcenode.py
index ef29c3c7..f72010f5 100755
--- a/onionr/communicatorutils/announcenode.py
+++ b/onionr/communicatorutils/announcenode.py
@@ -20,6 +20,7 @@
 import base64
 import onionrproofs, logger
 from etc import onionrvalues
+from onionrutils import basicrequests
 
 def announce_node(daemon):
     '''Announce our node to our peers'''
@@ -75,8 +76,8 @@ def announce_node(daemon):
                     daemon.announceCache[peer] = data['random']
             if not announceFail:
                 logger.info('Announcing node to ' + url)
-                if daemon._core._utils.doPostRequest(url, data) == 'Success':
-                    logger.info('Successfully introduced node to ' + peer)
+                if basicrequests.do_post_request(daemon._core, url, data) == 'Success':
+                    logger.info('Successfully introduced node to ' + peer, terminal=True)
                     retData = True
                     daemon._core.setAddressInfo(peer, 'introduced', 1)
                     daemon._core.setAddressInfo(peer, 'powValue', data['random'])
diff --git a/onionr/communicatorutils/connectnewpeers.py b/onionr/communicatorutils/connectnewpeers.py
index 25929c24..c3a9f77b 100755
--- a/onionr/communicatorutils/connectnewpeers.py
+++ b/onionr/communicatorutils/connectnewpeers.py
@@ -20,7 +20,7 @@
 import time, sys
 import onionrexceptions, logger, onionrpeers
 from utils import networkmerger
-from onionrutils import stringvalidators
+from onionrutils import stringvalidators, epoch
 # secrets module was added into standard lib in 3.6+
 if sys.version_info[0] == 3 and sys.version_info[1] < 6:
     from dependencies import secrets
@@ -75,7 +75,7 @@ def connect_new_peer_to_communicator(comm_inst, peer='', useBootstrap=False):
             if address not in comm_inst.onlinePeers:
                 logger.info('Connected to ' + address, terminal=True)
                 comm_inst.onlinePeers.append(address)
-                comm_inst.connectTimes[address] = comm_inst._core._utils.getEpoch()
+                comm_inst.connectTimes[address] = epoch.get_epoch()
             retData = address
 
             # add peer to profile list if they're not in it
diff --git a/onionr/communicatorutils/cooldownpeer.py b/onionr/communicatorutils/cooldownpeer.py
index 26cf2dd9..33bcc277 100755
--- a/onionr/communicatorutils/cooldownpeer.py
+++ b/onionr/communicatorutils/cooldownpeer.py
@@ -17,6 +17,7 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
+from onionrutils import epoch
 def cooldown_peer(comm_inst):
     '''Randomly add an online peer to cooldown, so we can connect a new one'''
     onlinePeerAmount = len(comm_inst.onlinePeers)
@@ -28,7 +29,7 @@ def cooldown_peer(comm_inst):
     # Remove peers from cooldown that have been there long enough
     tempCooldown = dict(comm_inst.cooldownPeer)
     for peer in tempCooldown:
-        if (comm_inst._core._utils.getEpoch() - tempCooldown[peer]) >= cooldownTime:
+        if (epoch.get_epoch() - tempCooldown[peer]) >= cooldownTime:
             del comm_inst.cooldownPeer[peer]
 
     # Cool down a peer, if we have max connections alive for long enough
@@ -38,7 +39,7 @@ def cooldown_peer(comm_inst):
         while finding:
             try:
                 toCool = min(tempConnectTimes, key=tempConnectTimes.get)
-                if (comm_inst._core._utils.getEpoch() - tempConnectTimes[toCool]) < minTime:
+                if (epoch.get_epoch() - tempConnectTimes[toCool]) < minTime:
                     del tempConnectTimes[toCool]
                 else:
                     finding = False
@@ -46,6 +47,6 @@ def cooldown_peer(comm_inst):
                 break
         else:
             comm_inst.removeOnlinePeer(toCool)
-            comm_inst.cooldownPeer[toCool] = comm_inst._core._utils.getEpoch()
+            comm_inst.cooldownPeer[toCool] = epoch.get_epoch()
 
     comm_inst.decrementThreadCount('cooldown_peer')
\ No newline at end of file
diff --git a/onionr/communicatorutils/downloadblocks.py b/onionr/communicatorutils/downloadblocks.py
index ccf121c5..ecd7b873 100755
--- a/onionr/communicatorutils/downloadblocks.py
+++ b/onionr/communicatorutils/downloadblocks.py
@@ -19,7 +19,7 @@
 '''
 import communicator, onionrexceptions
 import logger, onionrpeers
-from onionrutils import blockmetadata
+from onionrutils import blockmetadata, stringvalidators, validatemetadata
 
 def download_blocks_from_communicator(comm_inst):
     assert isinstance(comm_inst, communicator.OnionrCommunicatorDaemon)
@@ -48,7 +48,7 @@ def download_blocks_from_communicator(comm_inst):
             continue
         if comm_inst._core._blacklist.inBlacklist(blockHash):
             continue
-        if comm_inst._core._utils.storageCounter.isFull():
+        if comm_inst._core.storage_counter.isFull():
             break
         comm_inst.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block
         if len(blockPeers) == 0:
@@ -75,7 +75,7 @@ def download_blocks_from_communicator(comm_inst):
                 content = content.decode() # decode here because sha3Hash needs bytes above
                 metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
                 metadata = metas[0]
-                if comm_inst._core._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid, and verify nonce
+                if validatemetadata.validate_metadata(comm_inist._core, metadata, metas[2]): # check if metadata is valid, and verify nonce
                     if comm_inst._core._crypto.verifyPow(content): # check if POW is enough/correct
                         logger.info('Attempting to save block %s...' % blockHash[:12])
                         try:
diff --git a/onionr/communicatorutils/housekeeping.py b/onionr/communicatorutils/housekeeping.py
index 829564ab..3071a994 100755
--- a/onionr/communicatorutils/housekeeping.py
+++ b/onionr/communicatorutils/housekeeping.py
@@ -20,6 +20,7 @@
 import sqlite3
 import logger
 from onionrusers import onionrusers
+from onionrutils import epoch
 def clean_old_blocks(comm_inst):
     '''Delete old blocks if our disk allocation is full/near full, and also expired blocks'''
 
@@ -29,7 +30,7 @@ def clean_old_blocks(comm_inst):
         comm_inst._core.removeBlock(bHash)
         logger.info('Deleted block: %s' % (bHash,))
 
-    while comm_inst._core._utils.storageCounter.isFull():
+    while comm_inst._core.storage_counter.isFull():
         oldest = comm_inst._core.getBlockList()[0]
         comm_inst._core._blacklist.addToDB(oldest)
         comm_inst._core.removeBlock(oldest)
@@ -41,7 +42,7 @@ def clean_keys(comm_inst):
     '''Delete expired forward secrecy keys'''
     conn = sqlite3.connect(comm_inst._core.peerDB, timeout=10)
     c = conn.cursor()
-    time = comm_inst._core._utils.getEpoch()
+    time = epoch.get_epoch()
     deleteKeys = []
 
     for entry in c.execute("SELECT * FROM forwardKeys WHERE expire <= ?", (time,)):
diff --git a/onionr/communicatorutils/lookupblocks.py b/onionr/communicatorutils/lookupblocks.py
index 490a051e..7fae6522 100755
--- a/onionr/communicatorutils/lookupblocks.py
+++ b/onionr/communicatorutils/lookupblocks.py
@@ -18,6 +18,8 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 import logger, onionrproofs
+from onionrutils import stringvalidators, epoch
+
 def lookup_blocks_from_communicator(comm_inst):
     logger.info('Looking up new blocks...')
     tryAmount = 2
@@ -34,7 +36,7 @@ def lookup_blocks_from_communicator(comm_inst):
         if not comm_inst.isOnline:
             break
         # check if disk allocation is used
-        if comm_inst._core._utils.storageCounter.isFull():
+        if comm_inst._core.storage_counter.isFull():
             logger.debug('Not looking up new blocks due to maximum amount of allowed disk space used')
             break
         peer = comm_inst.pickOnlinePeer() # select random online peer
@@ -60,11 +62,11 @@ def lookup_blocks_from_communicator(comm_inst):
             logger.warn('Could not get new blocks from %s.' % peer, error = error)
             newBlocks = False
         else:
-            comm_inst.dbTimestamps[peer] = comm_inst._core._utils.getRoundedEpoch(roundS=60)
+            comm_inst.dbTimestamps[peer] = epoch.get_rounded_epoch(roundS=60)
         if newBlocks != False:
             # if request was a success
             for i in newBlocks.split('\n'):
-                if comm_inst._core._utils.validateHash(i):
+                if stringvalidators.validate_hash(i):
                     # if newline seperated string is valid hash
                     if not i in existingBlocks:
                         # if block does not exist on disk and is not already in block queue
diff --git a/onionr/communicatorutils/netcheck.py b/onionr/communicatorutils/netcheck.py
index 688feeea..85d10605 100755
--- a/onionr/communicatorutils/netcheck.py
+++ b/onionr/communicatorutils/netcheck.py
@@ -20,14 +20,14 @@
 '''
 import logger
 from utils import netutils
-from onionrutils import localcommand
+from onionrutils import localcommand, epoch
 def net_check(comm_inst):
     '''Check if we are connected to the internet or not when we can't connect to any peers'''
     rec = False # for detecting if we have received incoming connections recently
     c = comm_inst._core
     if len(comm_inst.onlinePeers) == 0:
         try:
-            if (c._utils.getEpoch() - int(localcommand.local_command(c, '/lastconnect'))) <= 60:
+            if (epoch.get_epoch() - int(localcommand.local_command(c, '/lastconnect'))) <= 60:
                 comm_inst.isOnline = True
                 rec = True
         except ValueError:
diff --git a/onionr/communicatorutils/uploadblocks.py b/onionr/communicatorutils/uploadblocks.py
index 497e07d5..afe96525 100755
--- a/onionr/communicatorutils/uploadblocks.py
+++ b/onionr/communicatorutils/uploadblocks.py
@@ -20,16 +20,17 @@
 import logger
 from communicatorutils import proxypicker
 import onionrblockapi as block
-from onionrutils import localcommand
+from onionrutils import localcommand, stringvalidators, basicrequests
 
 def upload_blocks_from_communicator(comm_inst):
     # when inserting a block, we try to upload it to a few peers to add some deniability
     triedPeers = []
     finishedUploads = []
-    comm_inst.blocksToUpload = comm_inst._core._crypto.randomShuffle(comm_inst.blocksToUpload)
+    core = comm_inst._core
+    comm_inst.blocksToUpload = core._crypto.randomShuffle(comm_inst.blocksToUpload)
     if len(comm_inst.blocksToUpload) != 0:
         for bl in comm_inst.blocksToUpload:
-            if not comm_inst._core._utils.validateHash(bl):
+            if not stringvalidators.validate_hash(bl):
                 logger.warn('Requested to upload invalid block')
                 comm_inst.decrementThreadCount('uploadBlock')
                 return
@@ -42,8 +43,8 @@ def upload_blocks_from_communicator(comm_inst):
                 data = {'block': block.Block(bl).getRaw()}
                 proxyType = proxypicker.pick_proxy(peer)
                 logger.info("Uploading block to " + peer)
-                if not comm_inst._core._utils.doPostRequest(url, data=data, proxyType=proxyType) == False:
-                    localcommand.local_command(comm_inst._core, 'waitforshare/' + bl, post=True)
+                if not basicrequests.do_post_request(core, url, data=data, proxyType=proxyType) == False:
+                    localcommand.local_command(core, 'waitforshare/' + bl, post=True)
                     finishedUploads.append(bl)
     for x in finishedUploads:
         try:
diff --git a/onionr/core.py b/onionr/core.py
index 96068a66..0d64877a 100755
--- a/onionr/core.py
+++ b/onionr/core.py
@@ -30,6 +30,7 @@ import dbcreator, onionrstorage, serializeddata, subprocesspow
 from etc import onionrvalues, powchoice
 from onionrutils import localcommand, stringvalidators, bytesconverter, epoch
 from onionrutils import blockmetadata
+import storagecounter
 
 class Core:
     def __init__(self, torPort=0):
@@ -41,76 +42,76 @@ class Core:
         if not self.dataDir.endswith('/'):
             self.dataDir += '/'
 
-        try:
-            self.onionrInst = None
-            self.queueDB = self.dataDir + 'queue.db'
-            self.peerDB = self.dataDir + 'peers.db'
-            self.blockDB = self.dataDir + 'blocks.db'
-            self.blockDataLocation = self.dataDir + 'blocks/'
-            self.blockDataDB = self.blockDataLocation + 'block-data.db'
-            self.publicApiHostFile = self.dataDir + 'public-host.txt'
-            self.privateApiHostFile = self.dataDir + 'private-host.txt'
-            self.addressDB = self.dataDir + 'address.db'
-            self.hsAddress = ''
-            self.i2pAddress = config.get('i2p.own_addr', None)
-            self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt'
-            self.bootstrapList = []
-            self.requirements = onionrvalues.OnionrValues()
-            self.torPort = torPort
-            self.dataNonceFile = self.dataDir + 'block-nonces.dat'
-            self.dbCreate = dbcreator.DBCreator(self)
-            self.forwardKeysFile = self.dataDir + 'forward-keys.db'
-            self.keyStore = simplekv.DeadSimpleKV(self.dataDir + 'cachedstorage.dat', refresh_seconds=5)
-            
-            # Socket data, defined here because of multithreading constraints with gevent
-            self.killSockets = False
-            self.startSocket = {}
-            self.socketServerConnData = {}
-            self.socketReasons = {}
-            self.socketServerResponseData = {}
+        #try:
+        self.usageFile = self.dataDir + 'disk-usage.txt'
+        self.config = config
+        self.maxBlockSize = 10000000 # max block size in bytes
 
-            self.usageFile = self.dataDir + 'disk-usage.txt'
-            self.config = config
+        self.onionrInst = None
+        self.queueDB = self.dataDir + 'queue.db'
+        self.peerDB = self.dataDir + 'peers.db'
+        self.blockDB = self.dataDir + 'blocks.db'
+        self.blockDataLocation = self.dataDir + 'blocks/'
+        self.blockDataDB = self.blockDataLocation + 'block-data.db'
+        self.publicApiHostFile = self.dataDir + 'public-host.txt'
+        self.privateApiHostFile = self.dataDir + 'private-host.txt'
+        self.addressDB = self.dataDir + 'address.db'
+        self.hsAddress = ''
+        self.i2pAddress = config.get('i2p.own_addr', None)
+        self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt'
+        self.bootstrapList = []
+        self.requirements = onionrvalues.OnionrValues()
+        self.torPort = torPort
+        self.dataNonceFile = self.dataDir + 'block-nonces.dat'
+        self.dbCreate = dbcreator.DBCreator(self)
+        self.forwardKeysFile = self.dataDir + 'forward-keys.db'
+        self.keyStore = simplekv.DeadSimpleKV(self.dataDir + 'cachedstorage.dat', refresh_seconds=5)
+        self.storage_counter = storagecounter.StorageCounter(self)
+        
+        # Socket data, defined here because of multithreading constraints with gevent
+        self.killSockets = False
+        self.startSocket = {}
+        self.socketServerConnData = {}
+        self.socketReasons = {}
+        self.socketServerResponseData = {}
 
-            self.maxBlockSize = 10000000 # max block size in bytes
+        if not os.path.exists(self.dataDir):
+            os.mkdir(self.dataDir)
+        if not os.path.exists(self.dataDir + 'blocks/'):
+            os.mkdir(self.dataDir + 'blocks/')
+        if not os.path.exists(self.blockDB):
+            self.createBlockDB()
+        if not os.path.exists(self.forwardKeysFile):
+            self.dbCreate.createForwardKeyDB()
+        if not os.path.exists(self.peerDB):
+            self.createPeerDB()
+        if not os.path.exists(self.addressDB):
+            self.createAddressDB()
 
-            if not os.path.exists(self.dataDir):
-                os.mkdir(self.dataDir)
-            if not os.path.exists(self.dataDir + 'blocks/'):
-                os.mkdir(self.dataDir + 'blocks/')
-            if not os.path.exists(self.blockDB):
-                self.createBlockDB()
-            if not os.path.exists(self.forwardKeysFile):
-                self.dbCreate.createForwardKeyDB()
-            if not os.path.exists(self.peerDB):
-                self.createPeerDB()
-            if not os.path.exists(self.addressDB):
-                self.createAddressDB()
+        if os.path.exists(self.dataDir + '/hs/hostname'):
+            with open(self.dataDir + '/hs/hostname', 'r') as hs:
+                self.hsAddress = hs.read().strip()
 
-            if os.path.exists(self.dataDir + '/hs/hostname'):
-                with open(self.dataDir + '/hs/hostname', 'r') as hs:
-                    self.hsAddress = hs.read().strip()
+        # Load bootstrap address list
+        if os.path.exists(self.bootstrapFileLocation):
+            with open(self.bootstrapFileLocation, 'r') as bootstrap:
+                bootstrap = bootstrap.read()
+            for i in bootstrap.split('\n'):
+                self.bootstrapList.append(i)
+        else:
+            logger.warn('Warning: address bootstrap file not found ' + self.bootstrapFileLocation)
 
-            # Load bootstrap address list
-            if os.path.exists(self.bootstrapFileLocation):
-                with open(self.bootstrapFileLocation, 'r') as bootstrap:
-                    bootstrap = bootstrap.read()
-                for i in bootstrap.split('\n'):
-                    self.bootstrapList.append(i)
-            else:
-                logger.warn('Warning: address bootstrap file not found ' + self.bootstrapFileLocation)
+        self.use_subprocess = powchoice.use_subprocess(self)
+        # Initialize the crypto object
+        self._crypto = onionrcrypto.OnionrCrypto(self)
+        self._blacklist = onionrblacklist.OnionrBlackList(self)
+        self.serializer = serializeddata.SerializedData(self)
 
-            self.use_subprocess = powchoice.use_subprocess(self)
-            self._utils = onionrutils.OnionrUtils(self)
-            # Initialize the crypto object
-            self._crypto = onionrcrypto.OnionrCrypto(self)
-            self._blacklist = onionrblacklist.OnionrBlackList(self)
-            self.serializer = serializeddata.SerializedData(self)
-
-        except Exception as error:
-            logger.error('Failed to initialize core Onionr library.', error=error)
-            logger.fatal('Cannot recover from error.')
-            sys.exit(1)
+        # except Exception as error:
+        #     print(str(error))
+        #     logger.error('Failed to initialize core Onionr library.', error=error, terminal=True)
+        #     logger.fatal('Cannot recover from error.', terminal=True)
+        #     sys.exit(1)
         return
 
     def refreshFirstStartVars(self):
@@ -313,7 +314,7 @@ class Core:
             encryptType must be specified to encrypt a block
         '''
         allocationReachedMessage = 'Cannot insert block, disk allocation reached.'
-        if self._utils.storageCounter.isFull():
+        if self.storage_counter.isFull():
             logger.error(allocationReachedMessage)
             return False
         retData = False
@@ -439,7 +440,7 @@ class Core:
                     localcommand.local_command(self, '/waitforshare/' + retData, post=True, maxWait=5)
                     self.daemonQueueAdd('uploadBlock', retData)
                 self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
-                blockmetadata.process_block_metadata(retData)
+                blockmetadata.process_block_metadata(self, retData)
 
         if retData != False:
             if plaintextPeer == onionrvalues.DENIABLE_PEER_ADDRESS:
diff --git a/onionr/coredb/blockmetadb/add.py b/onionr/coredb/blockmetadb/add.py
index 69ac9b27..bdf9457f 100644
--- a/onionr/coredb/blockmetadb/add.py
+++ b/onionr/coredb/blockmetadb/add.py
@@ -1,5 +1,5 @@
 import os, sqlite3
-import onionrutils
+from onionrutils import epoch, blockmetadata
 def add_to_block_DB(core_inst, newHash, selfInsert=False, dataSaved=False):
     '''
         Add a hash value to the block db
@@ -9,11 +9,11 @@ def add_to_block_DB(core_inst, newHash, selfInsert=False, dataSaved=False):
 
     if not os.path.exists(core_inst.blockDB):
         raise Exception('Block db does not exist')
-    if onionrutils.has_block(core_inst, newHash):
+    if blockmetadata.has_block(core_inst, newHash):
         return
     conn = sqlite3.connect(core_inst.blockDB, timeout=30)
     c = conn.cursor()
-    currentTime = core_inst._utils.getEpoch() + core_inst._crypto.secrets.randbelow(301)
+    currentTime = epoch.get_epoch() + core_inst._crypto.secrets.randbelow(301)
     if selfInsert or dataSaved:
         selfInsert = 1
     else:
diff --git a/onionr/coredb/blockmetadb/expiredblocks.py b/onionr/coredb/blockmetadb/expiredblocks.py
index 8debbe26..0b102fe6 100644
--- a/onionr/coredb/blockmetadb/expiredblocks.py
+++ b/onionr/coredb/blockmetadb/expiredblocks.py
@@ -1,9 +1,10 @@
 import sqlite3
+from onionrutils import epoch
 def get_expired_blocks(core_inst):
     '''Returns a list of expired blocks'''
     conn = sqlite3.connect(core_inst.blockDB, timeout=30)
     c = conn.cursor()
-    date = int(core_inst._utils.getEpoch())
+    date = int(epoch.get_epoch())
 
     execute = 'SELECT hash FROM hashes WHERE expire <= %s ORDER BY dateReceived;' % (date,)
 
diff --git a/onionr/coredb/daemonqueue/__init__.py b/onionr/coredb/daemonqueue/__init__.py
index a2f7a483..252cbc37 100644
--- a/onionr/coredb/daemonqueue/__init__.py
+++ b/onionr/coredb/daemonqueue/__init__.py
@@ -1,6 +1,6 @@
 import sqlite3, os
 import onionrevents as events
-from onionrutils import localcommand
+from onionrutils import localcommand, epoch
 
 def daemon_queue(core_inst):
     '''
@@ -38,7 +38,7 @@ def daemon_queue_add(core_inst, command, data='', responseID=''):
 
     retData = True
 
-    date = core_inst._utils.getEpoch()
+    date = epoch.get_epoch()
     conn = sqlite3.connect(core_inst.queueDB, timeout=30)
     c = conn.cursor()
     t = (command, data, date, responseID)
diff --git a/onionr/coredb/keydb/listkeys.py b/onionr/coredb/keydb/listkeys.py
index 5ab44fb9..3ba23678 100644
--- a/onionr/coredb/keydb/listkeys.py
+++ b/onionr/coredb/keydb/listkeys.py
@@ -1,5 +1,6 @@
 import sqlite3
 import logger
+from onionrutils import epoch
 def list_peers(core_inst, randomOrder=True, getPow=False, trust=0):
     '''
         Return a list of public keys (misleading function name)
@@ -56,7 +57,7 @@ def list_adders(core_inst, randomOrder=True, i2p=True, recent=0):
     testList = list(addressList) # create new list to iterate
     for address in testList:
         try:
-            if recent > 0 and (core_inst._utils.getEpoch() - core_inst.getAddressInfo(address, 'lastConnect')) > recent:
+            if recent > 0 and (epoch.get_epoch() - core_inst.getAddressInfo(address, 'lastConnect')) > recent:
                 raise TypeError # If there is no last-connected date or it was too long ago, don't add peer to list if recent is not 0
         except TypeError:
             addressList.remove(address)
diff --git a/onionr/httpapi/miscpublicapi/getblocks.py b/onionr/httpapi/miscpublicapi/getblocks.py
index c1e58e4f..08d9f678 100755
--- a/onionr/httpapi/miscpublicapi/getblocks.py
+++ b/onionr/httpapi/miscpublicapi/getblocks.py
@@ -18,7 +18,8 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 from flask import Response, abort
-import config, onionrutils
+import config
+from onionrutils import bytesconverter, stringvalidators
 def get_public_block_list(clientAPI, publicAPI, request):
     # Provide a list of our blocks, with a date offset
     dateAdjust = request.args.get('date')
@@ -33,7 +34,7 @@ def get_public_block_list(clientAPI, publicAPI, request):
 def get_block_data(clientAPI, publicAPI, data):
     '''data is the block hash in hex'''
     resp = ''
-    if clientAPI._utils.validateHash(data):
+    if stringvalidators.validate_hash(data):
         if not clientAPI._core.config.get('general.hide_created_blocks', True) or data not in publicAPI.hideBlocks:
             if data in clientAPI._core.getBlockList():
                 block = clientAPI.getBlockData(data, raw=True)
@@ -41,7 +42,7 @@ def get_block_data(clientAPI, publicAPI, data):
                     block = block.encode() # Encode in case data is binary
                 except AttributeError:
                     abort(404)
-                block = onionrutils.str_to_bytes(block)
+                block = bytesconverter.str_to_bytes(block)
                 resp = block
     if len(resp) == 0:
         abort(404)
diff --git a/onionr/keymanager.py b/onionr/keymanager.py
index 9e2a1703..a288d87e 100755
--- a/onionr/keymanager.py
+++ b/onionr/keymanager.py
@@ -17,20 +17,20 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
+from onionrutils import bytesconverter
 import onionrcrypto
 class KeyManager:
     def __init__(self, crypto):
         assert isinstance(crypto, onionrcrypto.OnionrCrypto)
         self._core = crypto._core
-        self._utils = self._core._utils
         self.keyFile = crypto._keyFile
         self.crypto = crypto
 
     def addKey(self, pubKey=None, privKey=None):
         if type(pubKey) is type(None) and type(privKey) is type(None):
             pubKey, privKey = self.crypto.generatePubKey()
-        pubKey = self.crypto._core._utils.bytesToStr(pubKey)
-        privKey = self.crypto._core._utils.bytesToStr(privKey)
+        pubKey = bytesconverter.bytes_to_str(pubKey)
+        privKey = bytesconverter.bytes_to_str(privKey)
         try:
             if pubKey in self.getPubkeyList():
                 raise ValueError('Pubkey already in list: %s' % (pubKey,))
diff --git a/onionr/onionr.py b/onionr/onionr.py
index 74c0c335..85a34020 100755
--- a/onionr/onionr.py
+++ b/onionr/onionr.py
@@ -32,7 +32,6 @@ if sys.version_info[0] == 2 or sys.version_info[1] < MIN_PY_VERSION:
 import os, base64, random, shutil, time, platform, signal
 from threading import Thread
 import api, core, config, logger, onionrplugins as plugins, onionrevents as events
-import onionrutils
 import netcontroller
 from netcontroller import NetController
 from onionrblockapi import Block
@@ -51,6 +50,7 @@ class Onionr:
             Main Onionr class. This is for the CLI program, and does not handle much of the logic.
             In general, external programs and plugins should not use this class.
         '''
+        self.API_VERSION = API_VERSION
         self.userRunDir = os.getcwd() # Directory user runs the program from
         self.killed = False
 
diff --git a/onionr/onionrblacklist.py b/onionr/onionrblacklist.py
index 9c4e99d7..03da61b4 100755
--- a/onionr/onionrblacklist.py
+++ b/onionr/onionrblacklist.py
@@ -18,6 +18,7 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 import sqlite3, os, logger
+from onionrutils import epoch, bytesconverter
 class OnionrBlackList:
     def __init__(self, coreInst):
         self.blacklistDB = coreInst.dataDir + 'blacklist.db'
@@ -28,7 +29,7 @@ class OnionrBlackList:
         return
 
     def inBlacklist(self, data):
-        hashed = self._core._utils.bytesToStr(self._core._crypto.sha3Hash(data))
+        hashed = bytesconverter.bytes_to_str(self._core._crypto.sha3Hash(data))
         retData = False
 
         if not hashed.isalnum():
@@ -56,7 +57,7 @@ class OnionrBlackList:
     def deleteExpired(self, dataType=0):
         '''Delete expired entries'''
         deleteList = []
-        curTime = self._core._utils.getEpoch()
+        curTime = epoch.get_epoch()
 
         try:
             int(dataType)
@@ -98,7 +99,7 @@ class OnionrBlackList:
         2=pubkey
         '''
         # we hash the data so we can remove data entirely from our node's disk
-        hashed = self._core._utils.bytesToStr(self._core._crypto.sha3Hash(data))
+        hashed = bytesconverter.bytes_to_str(self._core._crypto.sha3Hash(data))
         if len(hashed) > 64:
             raise Exception("Hashed data is too large")
 
@@ -115,7 +116,7 @@ class OnionrBlackList:
         if self.inBlacklist(hashed):
             return
         insert = (hashed,)
-        blacklistDate = self._core._utils.getEpoch()
+        blacklistDate = epoch.get_epoch()
         try:
             self._dbExecute("INSERT INTO blacklist (hash, dataType, blacklistDate, expire) VALUES(?, ?, ?, ?);", (str(hashed), dataType, blacklistDate, expire))
         except sqlite3.IntegrityError:
diff --git a/onionr/onionrblockapi.py b/onionr/onionrblockapi.py
index cc823a1f..8464659f 100755
--- a/onionr/onionrblockapi.py
+++ b/onionr/onionrblockapi.py
@@ -21,7 +21,7 @@
 import core as onionrcore, logger, config, onionrexceptions, nacl.exceptions
 import json, os, sys, datetime, base64, onionrstorage
 from onionrusers import onionrusers
-from onionrutils import stringvalidators
+from onionrutils import stringvalidators, epoch
 
 class Block:
     blockCacheOrder = list() # NEVER write your own code that writes to this!
@@ -89,7 +89,7 @@ class Block:
 
                 # Check for replay attacks
                 try:
-                    if self.core._utils.getEpoch() - self.core.getBlockDate(self.hash) < 60:
+                    if epoch.get_epoch() - self.core.getBlockDate(self.hash) < 60:
                         assert self.core._crypto.replayTimestampValidation(self.bmetadata['rply'])
                 except (AssertionError, KeyError, TypeError) as e:
                     if not self.bypassReplayCheck:
diff --git a/onionr/onionrcommands/banblocks.py b/onionr/onionrcommands/banblocks.py
index fe50d16a..426e852e 100755
--- a/onionr/onionrcommands/banblocks.py
+++ b/onionr/onionrcommands/banblocks.py
@@ -19,12 +19,13 @@
 '''
 import sys
 import logger
+from onionrutils import stringvalidators
 def ban_block(o_inst):
     try:
         ban = sys.argv[2]
     except IndexError:
         ban = logger.readline('Enter a block hash:')
-    if o_inst.onionrUtils.validateHash(ban):
+    if stringvalidators.validate_hash(ban):
         if not o_inst.onionrCore._blacklist.inBlacklist(ban):
             try:
                 o_inst.onionrCore._blacklist.addToDB(ban)
diff --git a/onionr/onionrcommands/exportblocks.py b/onionr/onionrcommands/exportblocks.py
index daa70f35..a941e025 100755
--- a/onionr/onionrcommands/exportblocks.py
+++ b/onionr/onionrcommands/exportblocks.py
@@ -19,6 +19,7 @@
 '''
 import sys, os
 import logger, onionrstorage
+from onionrutils import stringvalidators
 def doExport(o_inst, bHash):
     exportDir = o_inst.dataDir + 'block-export/'
     if not os.path.exists(exportDir):
@@ -34,7 +35,7 @@ def doExport(o_inst, bHash):
 def export_block(o_inst):
     exportDir = o_inst.dataDir + 'block-export/'
     try:
-        assert o_inst.onionrUtils.validateHash(sys.argv[2])
+        assert stringvalidators.validate_hash(sys.argv[2])
     except (IndexError, AssertionError):
         logger.error('No valid block hash specified.', terminal=True)
         sys.exit(1)
diff --git a/onionr/onionrcommands/filecommands.py b/onionr/onionrcommands/filecommands.py
index 8f97f90a..7e2cd086 100755
--- a/onionr/onionrcommands/filecommands.py
+++ b/onionr/onionrcommands/filecommands.py
@@ -21,6 +21,7 @@
 import base64, sys, os
 import logger
 from onionrblockapi import Block
+from onionrutils import stringvalidators
 def add_file(o_inst, singleBlock=False, blockType='bin'):
     '''
         Adds a file to the onionr network
@@ -60,7 +61,7 @@ def getFile(o_inst):
         if os.path.exists(fileName):
             logger.error("File already exists", terminal=True)
             return
-        if not o_inst.onionrUtils.validateHash(bHash):
+        if not stringvalidators.validate_hash(bHash):
             logger.error('Block hash is invalid', terminal=True)
             return
 
diff --git a/onionr/onionrcrypto.py b/onionr/onionrcrypto.py
index c6dfe300..957733eb 100755
--- a/onionr/onionrcrypto.py
+++ b/onionr/onionrcrypto.py
@@ -21,7 +21,7 @@ import os, binascii, base64, hashlib, time, sys, hmac, secrets
 import nacl.signing, nacl.encoding, nacl.public, nacl.hash, nacl.pwhash, nacl.utils, nacl.secret
 import unpaddedbase32
 import logger, onionrproofs
-from onionrutils import stringvalidators
+from onionrutils import stringvalidators, epoch, bytesconverter
 import onionrexceptions, keymanager, core, onionrutils
 import config
 config.reload()
@@ -95,10 +95,10 @@ class OnionrCrypto:
 
     def pubKeyEncrypt(self, data, pubkey, encodedData=False):
         '''Encrypt to a public key (Curve25519, taken from base32 Ed25519 pubkey)'''
-        pubkey = unpaddedbase32.repad(onionrutils.str_to_bytes(pubkey))
+        pubkey = unpaddedbase32.repad(bytesconverter.str_to_bytes(pubkey))
         retVal = ''
         box = None
-        data = onionrutils.str_to_bytes(data)
+        data = bytesconverter.str_to_bytes(data)
         
         pubkey = nacl.signing.VerifyKey(pubkey, encoder=nacl.encoding.Base32Encoder()).to_curve25519_public_key()
 
@@ -182,7 +182,7 @@ class OnionrCrypto:
     def generateDeterministic(self, passphrase, bypassCheck=False):
         '''Generate a Ed25519 public key pair from a password'''
         passStrength = self.deterministicRequirement
-        passphrase = onionrutils.str_to_bytes(passphrase) # Convert to bytes if not already
+        passphrase = bytesconverter.str_to_bytes(passphrase) # Convert to bytes if not already
         # Validate passphrase length
         if not bypassCheck:
             if len(passphrase) < passStrength:
@@ -202,7 +202,7 @@ class OnionrCrypto:
         if pubkey == '':
             pubkey = self.pubKey
         prev = ''
-        pubkey = onionrutils.str_to_bytes(pubkey)
+        pubkey = bytesconverter.str_to_bytes(pubkey)
         for i in range(self.HASH_ID_ROUNDS):
             try:
                 prev = prev.encode()
@@ -266,7 +266,7 @@ class OnionrCrypto:
 
     @staticmethod
     def replayTimestampValidation(timestamp):
-        if core.Core()._utils.getEpoch() - int(timestamp) > 2419200:
+        if epoch.get_epoch() - int(timestamp) > 2419200:
             return False
         else:
             return True
diff --git a/onionr/onionrpeers.py b/onionr/onionrpeers.py
index 8bf76289..d0c0c5cb 100755
--- a/onionr/onionrpeers.py
+++ b/onionr/onionrpeers.py
@@ -19,6 +19,7 @@
 '''
 import sqlite3
 import core, config, logger
+from onionrutils import epoch
 config.reload()
 class PeerProfiles:
     '''
@@ -106,7 +107,7 @@ def peerCleanup(coreInst):
             if PeerProfiles(address, coreInst).score < minScore:
                 coreInst.removeAddress(address)
                 try:
-                    if (int(coreInst._utils.getEpoch()) - int(coreInst.getPeerInfo(address, 'dateSeen'))) >= 600:
+                    if (int(epoch.get_epoch()) - int(coreInst.getPeerInfo(address, 'dateSeen'))) >= 600:
                         expireTime = 600
                     else:
                         expireTime = 86400
diff --git a/onionr/onionrpluginapi.py b/onionr/onionrpluginapi.py
index 24ba4f12..9b9842c3 100755
--- a/onionr/onionrpluginapi.py
+++ b/onionr/onionrpluginapi.py
@@ -170,9 +170,6 @@ class pluginapi:
     def get_core(self):
         return self.core
 
-    def get_utils(self):
-        return self.get_core()._utils
-
     def get_crypto(self):
         return self.get_core()._crypto
 
diff --git a/onionr/onionrproofs.py b/onionr/onionrproofs.py
index 7b8aa91a..51f35325 100755
--- a/onionr/onionrproofs.py
+++ b/onionr/onionrproofs.py
@@ -30,10 +30,7 @@ def getDifficultyModifier(coreOrUtilsInst=None):
     '''
     classInst = coreOrUtilsInst
     retData = 0
-    if isinstance(classInst, core.Core):
-        useFunc = classInst._utils.storageCounter.getPercent
-    else:
-        useFunc = core.Core()._utils.storageCounter.getPercent
+    useFunc = classInst.storage_counter.getPercent
 
     percentUse = useFunc()
 
diff --git a/onionr/onionrservices/__init__.py b/onionr/onionrservices/__init__.py
index 1c23e5fb..1f56dabe 100755
--- a/onionr/onionrservices/__init__.py
+++ b/onionr/onionrservices/__init__.py
@@ -21,7 +21,7 @@ import time
 import stem
 import core
 from . import connectionserver, bootstrapservice
-from onionrutils import stringvalidators
+from onionrutils import stringvalidators, basicrequests
 
 class OnionrServices:
     '''
@@ -47,7 +47,7 @@ class OnionrServices:
         base_url = 'http://%s/' % (address,)
         socks = self._core.config.get('tor.socksport')
         for x in range(BOOTSTRAP_TRIES):
-            if self._core._utils.doGetRequest(base_url + 'ping', port=socks, ignoreAPI=True) == 'pong!':
+            if basicrequests.do_get_request(self._core, base_url + 'ping', port=socks, ignoreAPI=True) == 'pong!':
                 # if bootstrap sever is online, tell them our service address
                 connectionserver.ConnectionServer(peer, address, core_inst=self._core)
             else:
diff --git a/onionr/onionrservices/bootstrapservice.py b/onionr/onionrservices/bootstrapservice.py
index 777801dd..752ce90b 100755
--- a/onionr/onionrservices/bootstrapservice.py
+++ b/onionr/onionrservices/bootstrapservice.py
@@ -24,7 +24,7 @@ from flask import Flask, Response
 import core
 from netcontroller import getOpenPort
 from . import httpheaders
-from onionrutils import stringvalidators
+from onionrutils import stringvalidators, epoch
 
 def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=300):
     '''
@@ -77,7 +77,7 @@ def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=300):
         # Create the v3 onion service
         response = controller.create_ephemeral_hidden_service({80: bootstrap_port}, key_type = 'NEW', key_content = 'ED25519-V3', await_publication = True)
         core_inst.insertBlock(response.service_id, header='con', sign=True, encryptType='asym', 
-        asymPeer=peer, disableForward=True, expire=(core_inst._utils.getEpoch() + bootstrap_timeout))
+        asymPeer=peer, disableForward=True, expire=(epoch.get_epoch() + bootstrap_timeout))
         # Run the bootstrap server
         try:
             http_server.serve_forever()
diff --git a/onionr/onionrservices/connectionserver.py b/onionr/onionrservices/connectionserver.py
index 3c4238a7..b9f70e56 100755
--- a/onionr/onionrservices/connectionserver.py
+++ b/onionr/onionrservices/connectionserver.py
@@ -24,7 +24,7 @@ import core, logger, httpapi
 import onionrexceptions
 from netcontroller import getOpenPort
 import api
-from onionrutils import stringvalidators
+from onionrutils import stringvalidators, basicrequests
 from . import httpheaders
 
 class ConnectionServer:
@@ -72,7 +72,7 @@ class ConnectionServer:
 
             try:
                 for x in range(3):
-                    attempt = self.core_inst._utils.doPostRequest('http://' + address + '/bs/' + response.service_id, port=socks)
+                    attempt = basicrequests.do_post_request(self.core_inst, 'http://' + address + '/bs/' + response.service_id, port=socks)
                     if attempt == 'success':
                         break
                 else:
diff --git a/onionr/onionrstorage/__init__.py b/onionr/onionrstorage/__init__.py
index 2af2661b..4bfc8bf6 100755
--- a/onionr/onionrstorage/__init__.py
+++ b/onionr/onionrstorage/__init__.py
@@ -18,7 +18,7 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 import core, sys, sqlite3, os, dbcreator, onionrexceptions
-from onionrutils import bytesconverter
+from onionrutils import bytesconverter, stringvalidators
 
 DB_ENTRY_SIZE_LIMIT = 10000 # Will be a config option
 
@@ -66,7 +66,7 @@ def deleteBlock(coreInst, blockHash):
 
 def store(coreInst, data, blockHash=''):
     assert isinstance(coreInst, core.Core)
-    assert coreInst._utils.validateHash(blockHash)
+    assert stringvalidators.validate_hash(blockHash)
     ourHash = coreInst._crypto.sha3Hash(data)
     if blockHash != '':
         assert ourHash == blockHash
@@ -81,7 +81,7 @@ def store(coreInst, data, blockHash=''):
 
 def getData(coreInst, bHash):
     assert isinstance(coreInst, core.Core)
-    assert coreInst._utils.validateHash(bHash)
+    assert stringvalidators.validate_hash(bHash)
 
     bHash = bytesconverter.bytes_to_str(bHash)
 
diff --git a/onionr/onionrstorage/removeblock.py b/onionr/onionrstorage/removeblock.py
index 23e574b6..76112199 100644
--- a/onionr/onionrstorage/removeblock.py
+++ b/onionr/onionrstorage/removeblock.py
@@ -1,5 +1,6 @@
 import sys, sqlite3
 import onionrexceptions, onionrstorage
+from onionrutils import stringvalidators
 def remove_block(core_inst, block):
     '''
         remove a block from this node (does not automatically blacklist)
@@ -7,7 +8,7 @@ def remove_block(core_inst, block):
         **You may want blacklist.addToDB(blockHash)
     '''
 
-    if core_inst._utils.validateHash(block):
+    if stringvalidators.validate_hash(block):
         conn = sqlite3.connect(core_inst.blockDB, timeout=30)
         c = conn.cursor()
         t = (block,)
@@ -15,6 +16,6 @@ def remove_block(core_inst, block):
         conn.commit()
         conn.close()
         dataSize = sys.getsizeof(onionrstorage.getData(core_inst, block))
-        core_inst._utils.storageCounter.removeBytes(dataSize)
+        core_inst.storage_counter.removeBytes(dataSize)
     else:
         raise onionrexceptions.InvalidHexHash
\ No newline at end of file
diff --git a/onionr/onionrstorage/setdata.py b/onionr/onionrstorage/setdata.py
index 60378112..50721ccf 100644
--- a/onionr/onionrstorage/setdata.py
+++ b/onionr/onionrstorage/setdata.py
@@ -19,7 +19,7 @@ def set_data(core_inst, data):
     try:
         onionrstorage.getData(core_inst, dataHash)
     except onionrexceptions.NoDataAvailable:
-        if core_inst._utils.storageCounter.addBytes(dataSize) != False:
+        if core_inst.storage_counter.addBytes(dataSize) != False:
             onionrstorage.store(core_inst, data, blockHash=dataHash)
             conn = sqlite3.connect(core_inst.blockDB, timeout=30)
             c = conn.cursor()
diff --git a/onionr/onionrusers/contactmanager.py b/onionr/onionrusers/contactmanager.py
index 1cac6953..680de9b0 100755
--- a/onionr/onionrusers/contactmanager.py
+++ b/onionr/onionrusers/contactmanager.py
@@ -20,7 +20,7 @@
 import os, json, onionrexceptions
 import unpaddedbase32
 from onionrusers import onionrusers
-from onionrutils import bytesconverter
+from onionrutils import bytesconverter, epoch
 
 class ContactManager(onionrusers.OnionrUser):
     def __init__(self, coreInst, publicKey, saveUser=False, recordExpireSeconds=5):
@@ -42,7 +42,7 @@ class ContactManager(onionrusers.OnionrUser):
             dataFile.write(data)
 
     def _loadData(self):
-        self.lastRead = self._core._utils.getEpoch()
+        self.lastRead = epoch.get_epoch()
         retData = {}
         if os.path.exists(self.dataFile):
             with open(self.dataFile, 'r') as dataFile:
@@ -62,7 +62,7 @@ class ContactManager(onionrusers.OnionrUser):
         if self.deleted:
             raise onionrexceptions.ContactDeleted
 
-        if (self._core._utils.getEpoch() - self.lastRead >= self.recordExpire) or forceReload:
+        if (epoch.get_epoch() - self.lastRead >= self.recordExpire) or forceReload:
             self.data = self._loadData()
         try:
             return self.data[key]
diff --git a/onionr/onionrusers/onionrusers.py b/onionr/onionrusers/onionrusers.py
index 5bb2eac8..28a5784d 100755
--- a/onionr/onionrusers/onionrusers.py
+++ b/onionr/onionrusers/onionrusers.py
@@ -18,8 +18,7 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 import logger, onionrexceptions, json, sqlite3, time
-from onionrutils import stringvalidators, bytesconverter
-
+from onionrutils import stringvalidators, bytesconverter, epoch
 import unpaddedbase32
 import nacl.exceptions
 
@@ -28,7 +27,7 @@ def deleteExpiredKeys(coreInst):
     conn = sqlite3.connect(coreInst.forwardKeysFile, timeout=10)
     c = conn.cursor()
 
-    curTime = coreInst._utils.getEpoch()
+    curTime = epoch.get_epoch()
     c.execute("DELETE from myForwardKeys where expire <= ?", (curTime,))
     conn.commit()
     conn.execute("VACUUM")
@@ -40,7 +39,7 @@ def deleteTheirExpiredKeys(coreInst, pubkey):
     c = conn.cursor()
 
     # Prepare the insert
-    command = (pubkey, coreInst._utils.getEpoch())
+    command = (pubkey, epoch.get_epoch())
 
     c.execute("DELETE from forwardKeys where peerKey = ? and expire <= ?", command)
 
@@ -160,10 +159,10 @@ class OnionrUser:
         conn = sqlite3.connect(self._core.forwardKeysFile, timeout=10)
         c = conn.cursor()
         # Prepare the insert
-        time = self._core._utils.getEpoch()
+        time = epoch.get_epoch()
         newKeys = self._core._crypto.generatePubKey()
-        newPub = self._core._utils.bytesToStr(newKeys[0])
-        newPriv = self._core._utils.bytesToStr(newKeys[1])
+        newPub = bytesconverter.bytes_to_str(newKeys[0])
+        newPriv = bytesconverter.bytes_to_str(newKeys[1])
 
         command = (self.publicKey, newPub, newPriv, time, expire + time)
 
@@ -178,7 +177,7 @@ class OnionrUser:
         conn = sqlite3.connect(self._core.forwardKeysFile, timeout=10)
         c = conn.cursor()
         pubkey = self.publicKey
-        pubkey = self._core._utils.bytesToStr(pubkey)
+        pubkey = bytesconverter.bytes_to_str(pubkey)
         command = (pubkey,)
         keyList = [] # list of tuples containing pub, private for peer
 
@@ -192,7 +191,7 @@ class OnionrUser:
         return list(keyList)
 
     def addForwardKey(self, newKey, expire=DEFAULT_KEY_EXPIRE):
-        newKey = self._core._utils.bytesToStr(unpaddedbase32.repad(bytesconverter.str_to_bytes(newKey)))
+        newKey = bytesconverter.bytes_to_str(unpaddedbase32.repad(bytesconverter.str_to_bytes(newKey)))
         if not stringvalidators.validate_pub_key(newKey):
             # Do not add if something went wrong with the key
             raise onionrexceptions.InvalidPubkey(newKey)
@@ -201,7 +200,7 @@ class OnionrUser:
         c = conn.cursor()
 
         # Get the time we're inserting the key at
-        timeInsert = self._core._utils.getEpoch()
+        timeInsert = epoch.get_epoch()
 
         # Look at our current keys for duplicate key data or time
         for entry in self._getForwardKeys():
diff --git a/onionr/onionrutils/__init__.py b/onionr/onionrutils/__init__.py
index e430a7b1..e69de29b 100644
--- a/onionr/onionrutils/__init__.py
+++ b/onionr/onionrutils/__init__.py
@@ -1,120 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    OnionrUtils offers various useful functions to Onionr. Relatively misc.
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-# Misc functions that do not fit in the main api, but are useful
-import sys, os, sqlite3, binascii, time, base64, json, glob, shutil, math, re, urllib.parse, string
-import requests
-import nacl.signing, nacl.encoding
-import unpaddedbase32
-import onionrexceptions, config, logger
-import onionrevents
-import storagecounter
-from etc import pgpwords, onionrvalues
-from . import localcommand, blockmetadata, basicrequests, validatemetadata
-from . import stringvalidators
-
-config.reload()
-class OnionrUtils:
-    '''
-        Various useful functions for validating things, etc functions, connectivity
-    '''
-    def __init__(self, coreInstance):
-        #self.fingerprintFile = 'data/own-fingerprint.txt' #TODO Remove since probably not needed
-        self._core = coreInstance # onionr core instance
-
-        self.avoidDupe = [] # list used to prevent duplicate requests per peer for certain actions
-        self.peerProcessing = {} # dict of current peer actions: peer, actionList
-        self.storageCounter = storagecounter.StorageCounter(self._core) # used to keep track of how much data onionr is using on disk
-        return
-
-    def escapeAnsi(self, line):
-        '''
-            Remove ANSI escape codes from a string with regex
-
-            taken or adapted from: https://stackoverflow.com/a/38662876 by user https://stackoverflow.com/users/802365/%c3%89douard-lopez
-            cc-by-sa-3 license https://creativecommons.org/licenses/by-sa/3.0/
-        '''
-        ansi_escape = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -/]*[@-~]')
-        return ansi_escape.sub('', line)
-
-    def validateHash(self, data, length=64):
-        '''
-            Validate if a string is a valid hash hex digest (does not compare, just checks length and charset)
-        '''
-        return stringvalidators.validate_hash(self, data, length)
-
-    def getEpoch(self):
-        '''returns epoch'''
-        return math.floor(time.time())
-
-    def doPostRequest(self, url, data={}, port=0, proxyType='tor'):
-        '''
-        Do a POST request through a local tor or i2p instance
-        '''
-        return basicrequests.do_post_request(self, url, data, port, proxyType)
-
-    def doGetRequest(self, url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=False):
-        '''
-        Do a get request through a local tor or i2p instance
-        '''
-        return basicrequests.do_get_request(self, url, port, proxyType, ignoreAPI, returnHeaders)
-
-def size(path='.'):
-    '''
-        Returns the size of a folder's contents in bytes
-    '''
-    total = 0
-    if os.path.exists(path):
-        if os.path.isfile(path):
-            total = os.path.getsize(path)
-        else:
-            for entry in os.scandir(path):
-                if entry.is_file():
-                    total += entry.stat().st_size
-                elif entry.is_dir():
-                    total += size(entry.path)
-    return total
-
-def humanSize(num, suffix='B'):
-    '''
-        Converts from bytes to a human readable format.
-    '''
-    for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
-        if abs(num) < 1024.0:
-            return "%.1f %s%s" % (num, unit, suffix)
-        num /= 1024.0
-    return "%.1f %s%s" % (num, 'Yi', suffix)
-
-def has_block(core_inst, hash):
-    '''
-        Check for new block in the list
-    '''
-    conn = sqlite3.connect(core_inst.blockDB)
-    c = conn.cursor()
-    if not stringvalidators.validate_hash(hash):
-        raise Exception("Invalid hash")
-    for result in c.execute("SELECT COUNT() FROM hashes WHERE hash = ?", (hash,)):
-        if result[0] >= 1:
-            conn.commit()
-            conn.close()
-            return True
-        else:
-            conn.commit()
-            conn.close()
-            return False
\ No newline at end of file
diff --git a/onionr/onionrutils/basicrequests.py b/onionr/onionrutils/basicrequests.py
index e889b887..1435db28 100644
--- a/onionr/onionrutils/basicrequests.py
+++ b/onionr/onionrutils/basicrequests.py
@@ -1,12 +1,12 @@
 import requests
 import logger, onionrexceptions
-def do_post_request(utils_inst, url, data={}, port=0, proxyType='tor'):
+def do_post_request(core_inst, url, data={}, port=0, proxyType='tor'):
     '''
     Do a POST request through a local tor or i2p instance
     '''
     if proxyType == 'tor':
         if port == 0:
-            port = utils_inst._core.torPort
+            port = core_inst.torPort
         proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
     elif proxyType == 'i2p':
         proxies = {'http': 'http://127.0.0.1:4444'}
@@ -24,11 +24,11 @@ def do_post_request(utils_inst, url, data={}, port=0, proxyType='tor'):
         retData = False
     return retData
 
-def do_get_request(utils_inst, url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=False):
+def do_get_request(core_inst, url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=False):
     '''
     Do a get request through a local tor or i2p instance
     '''
-    API_VERSION = utils_inst._core.onionrInst.API_VERSION
+    API_VERSION = core_inst.onionrInst.API_VERSION
     retData = False
     if proxyType == 'tor':
         if port == 0:
diff --git a/onionr/onionrutils/blockmetadata.py b/onionr/onionrutils/blockmetadata.py
index ace85190..d892257f 100644
--- a/onionr/onionrutils/blockmetadata.py
+++ b/onionr/onionrutils/blockmetadata.py
@@ -1,9 +1,9 @@
-import json
+import json, sqlite3
 import logger, onionrevents
 from onionrusers import onionrusers
 from etc import onionrvalues
 import onionrblockapi
-from . import epoch
+from . import epoch, stringvalidators, bytesconverter
 def get_block_metadata_from_data(blockData):
     '''
         accepts block contents as string, returns a tuple of 
@@ -33,24 +33,24 @@ def get_block_metadata_from_data(blockData):
         meta = metadata['meta']
     return (metadata, meta, data)
 
-def process_block_metadata(utils_inst, blockHash):
+def process_block_metadata(core_inst, blockHash):
     '''
         Read metadata from a block and cache it to the block database
     '''
     curTime = epoch.get_rounded_epoch(roundS=60)
-    myBlock = onionrblockapi.Block(blockHash, utils_inst._core)
+    myBlock = onionrblockapi.Block(blockHash, core_inst)
     if myBlock.isEncrypted:
         myBlock.decrypt()
     if (myBlock.isEncrypted and myBlock.decrypted) or (not myBlock.isEncrypted):
         blockType = myBlock.getMetadata('type') # we would use myBlock.getType() here, but it is bugged with encrypted blocks
-        signer = utils_inst.bytesToStr(myBlock.signer)
+        signer = bytesconverter.bytes_to_str(myBlock.signer)
         valid = myBlock.verifySig()
         if myBlock.getMetadata('newFSKey') is not None:
-            onionrusers.OnionrUser(utils_inst._core, signer).addForwardKey(myBlock.getMetadata('newFSKey'))
+            onionrusers.OnionrUser(core_inst, signer).addForwardKey(myBlock.getMetadata('newFSKey'))
             
         try:
             if len(blockType) <= 10:
-                utils_inst._core.updateBlockInfo(blockHash, 'dataType', blockType)
+                core_inst.updateBlockInfo(blockHash, 'dataType', blockType)
         except TypeError:
             logger.warn("Missing block information")
             pass
@@ -61,9 +61,28 @@ def process_block_metadata(utils_inst, blockHash):
         except (AssertionError, ValueError, TypeError) as e:
             expireTime = onionrvalues.OnionrValues().default_expire + curTime
         finally:
-            utils_inst._core.updateBlockInfo(blockHash, 'expire', expireTime)
+            core_inst.updateBlockInfo(blockHash, 'expire', expireTime)
         if not blockType is None:
-            utils_inst._core.updateBlockInfo(blockHash, 'dataType', blockType)
-        onionrevents.event('processblocks', data = {'block': myBlock, 'type': blockType, 'signer': signer, 'validSig': valid}, onionr = utils_inst._core.onionrInst)
+            core_inst.updateBlockInfo(blockHash, 'dataType', blockType)
+        onionrevents.event('processblocks', data = {'block': myBlock, 'type': blockType, 'signer': signer, 'validSig': valid}, onionr = core_inst.onionrInst)
     else:
-        pass
\ No newline at end of file
+        pass
+
+def has_block(core_inst, hash):
+    '''
+        Check for new block in the list
+    '''
+    conn = sqlite3.connect(core_inst.blockDB)
+    c = conn.cursor()
+    if not stringvalidators.validate_hash(hash):
+        raise Exception("Invalid hash")
+    for result in c.execute("SELECT COUNT() FROM hashes WHERE hash = ?", (hash,)):
+        if result[0] >= 1:
+            conn.commit()
+            conn.close()
+            return True
+        else:
+            conn.commit()
+            conn.close()
+            return False
+    return False
\ No newline at end of file
diff --git a/onionr/onionrutils/epoch.py b/onionr/onionrutils/epoch.py
index 1ee8ae25..0cda05cb 100644
--- a/onionr/onionrutils/epoch.py
+++ b/onionr/onionrutils/epoch.py
@@ -6,6 +6,6 @@ def get_rounded_epoch(roundS=60):
     epoch = get_epoch()
     return epoch - (epoch % roundS)
 
-def get_epoch(self):
+def get_epoch():
     '''returns epoch'''
     return math.floor(time.time())
\ No newline at end of file
diff --git a/onionr/onionrutils/escapeansi.py b/onionr/onionrutils/escapeansi.py
new file mode 100644
index 00000000..1cd5862a
--- /dev/null
+++ b/onionr/onionrutils/escapeansi.py
@@ -0,0 +1,10 @@
+import re
+def escape_ANSI(line):
+    '''
+        Remove ANSI escape codes from a string with regex
+
+        taken or adapted from: https://stackoverflow.com/a/38662876 by user https://stackoverflow.com/users/802365/%c3%89douard-lopez
+        cc-by-sa-3 license https://creativecommons.org/licenses/by-sa/3.0/
+    '''
+    ansi_escape = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -/]*[@-~]')
+    return ansi_escape.sub('', line)
\ No newline at end of file
diff --git a/onionr/onionrutils/stringvalidators.py b/onionr/onionrutils/stringvalidators.py
index cdfd2338..87fa4e8f 100644
--- a/onionr/onionrutils/stringvalidators.py
+++ b/onionr/onionrutils/stringvalidators.py
@@ -1,6 +1,7 @@
-import base64, string, onionrutils
+import base64, string
 import unpaddedbase32, nacl.signing, nacl.encoding
-def validate_hash(utils_inst, data, length=64):
+from onionrutils import bytesconverter
+def validate_hash(data, length=64):
     '''
         Validate if a string is a valid hash hex digest (does not compare, just checks length and charset)
     '''
@@ -25,7 +26,7 @@ def validate_pub_key(key):
     if type(key) is type(None):
         return False
     # Accept keys that have no = padding
-    key = unpaddedbase32.repad(onionrutils.str_to_bytes(key))
+    key = unpaddedbase32.repad(bytesconverter.str_to_bytes(key))
 
     retVal = False
     try:
diff --git a/onionr/onionrutils/validatemetadata.py b/onionr/onionrutils/validatemetadata.py
index aebd45e8..4e2e69f0 100644
--- a/onionr/onionrutils/validatemetadata.py
+++ b/onionr/onionrutils/validatemetadata.py
@@ -1,7 +1,7 @@
 import json
 import logger, onionrexceptions
 from etc import onionrvalues
-from onionrutils import stringvalidators
+from onionrutils import stringvalidators, epoch
 def validate_metadata(core_inst, metadata, blockData):
     '''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string'''
     # TODO, make this check sane sizes
@@ -37,18 +37,18 @@ def validate_metadata(core_inst, metadata, blockData):
                 if not stringvalidators.is_integer_string(metadata[i]):
                     logger.warn('Block metadata time stamp is not integer string or int')
                     break
-                isFuture = (metadata[i] - core_inst.getEpoch())
+                isFuture = (metadata[i] - epoch.get_epoch())
                 if isFuture > maxClockDifference:
                     logger.warn('Block timestamp is skewed to the future over the max %s: %s' (maxClockDifference, isFuture))
                     break
-                if (core_inst.getEpoch() - metadata[i]) > maxAge:
+                if (epoch.get_epoch() - metadata[i]) > maxAge:
                     logger.warn('Block is outdated: %s' % (metadata[i],))
                     break
             elif i == 'expire':
                 try:
-                    assert int(metadata[i]) > core_inst.getEpoch()
+                    assert int(metadata[i]) > epoch.get_epoch()
                 except AssertionError:
-                    logger.warn('Block is expired: %s less than %s' % (metadata[i], core_inst.getEpoch()))
+                    logger.warn('Block is expired: %s less than %s' % (metadata[i], epoch.get_epoch()))
                     break
             elif i == 'encryptType':
                 try:
diff --git a/onionr/static-data/default-plugins/esoteric/main.py b/onionr/static-data/default-plugins/esoteric/main.py
index 0c6ae8fa..495129c0 100755
--- a/onionr/static-data/default-plugins/esoteric/main.py
+++ b/onionr/static-data/default-plugins/esoteric/main.py
@@ -23,7 +23,7 @@ import locale, sys, os, threading, json
 locale.setlocale(locale.LC_ALL, '')
 import onionrservices, logger
 from onionrservices import bootstrapservice
-from onionrutils import stringvalidators
+from onionrutils import stringvalidators, epoch, basicrequests
 
 plugin_name = 'esoteric'
 PLUGIN_VERSION = '0.0.0'
@@ -58,8 +58,8 @@ class Esoteric:
                 else:
                     message += '\n'
             except EOFError:
-                message = json.dumps({'m': message, 't': self.myCore._utils.getEpoch()})
-                print(self.myCore._utils.doPostRequest('http://%s/esoteric/sendto' % (self.transport,), port=self.socks, data=message))
+                message = json.dumps({'m': message, 't': epoch.get_epoch()})
+                print(basicrequests.do_post_request(self.myCore, 'http://%s/esoteric/sendto' % (self.transport,), port=self.socks, data=message))
                 message = ''
             except KeyboardInterrupt:
                 self.shutdown = True
@@ -78,7 +78,7 @@ class Esoteric:
         self.socks = self.myCore.config.get('tor.socksport')
 
         print('connected with', peer, 'on', peer_transport_address)
-        if self.myCore._utils.doGetRequest('http://%s/ping' % (peer_transport_address,), ignoreAPI=True, port=self.socks) == 'pong!':
+        if basicrequests.do_get_request(self.myCore, 'http://%s/ping' % (peer_transport_address,), ignoreAPI=True, port=self.socks) == 'pong!':
             print('connected', peer_transport_address)
             threading.Thread(target=self._sender_loop).start()
 
diff --git a/onionr/static-data/default-plugins/flow/main.py b/onionr/static-data/default-plugins/flow/main.py
index 5f61aa38..de0c60e3 100755
--- a/onionr/static-data/default-plugins/flow/main.py
+++ b/onionr/static-data/default-plugins/flow/main.py
@@ -22,6 +22,7 @@
 import threading, time, locale, sys, os
 from onionrblockapi import Block
 import logger, config
+from onionrutils import escapeansi, epoch
 locale.setlocale(locale.LC_ALL, '')
 
 sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
@@ -43,7 +44,7 @@ class OnionrFlow:
         logger.warn("Please note: everything said here is public, even if a random channel name is used.", terminal=True)
         message = ""
         self.flowRunning = True
-        newThread = threading.Thread(target=self.showOutput)
+        newThread = threading.Thread(target=self.showOutput, daemon=True)
         newThread.start()
         try:
             self.channel = logger.readline("Enter a channel name or none for default:")
@@ -59,7 +60,7 @@ class OnionrFlow:
             else:
                 if message == "q":
                     self.flowRunning = False
-                expireTime = self.myCore._utils.getEpoch() + 43200
+                expireTime = epoch.get_epoch() + 43200
                 if len(message) > 0:
                     logger.info('Inserting message as block...', terminal=True)
                     self.myCore.insertBlock(message, header='txt', expire=expireTime, meta={'ch': self.channel})
@@ -83,7 +84,7 @@ class OnionrFlow:
                     logger.info('\n------------------------', prompt = False, terminal=True)
                     content = block.getContent()
                     # Escape new lines, remove trailing whitespace, and escape ansi sequences
-                    content = self.myCore._utils.escapeAnsi(content.replace('\n', '\\n').replace('\r', '\\r').strip())
+                    content = escapeansi.escape_ANSI(content.replace('\n', '\\n').replace('\r', '\\r').strip())
                     logger.info(block.getDate().strftime("%m/%d %H:%M") + ' - ' + logger.colors.reset + content, prompt = False, terminal=True)
                     self.alreadyOutputed.append(block.getHash())
                 time.sleep(5)
diff --git a/onionr/static-data/default-plugins/pluginmanager/main.py b/onionr/static-data/default-plugins/pluginmanager/main.py
index bb21d35c..807feec6 100755
--- a/onionr/static-data/default-plugins/pluginmanager/main.py
+++ b/onionr/static-data/default-plugins/pluginmanager/main.py
@@ -22,7 +22,7 @@
 import logger, config
 import os, sys, json, time, random, shutil, base64, getpass, datetime, re
 from onionrblockapi import Block
-from onionrutils import importnewblocks, stringvalidators, 
+from onionrutils import importnewblocks, stringvalidators
 
 plugin_name = 'pluginmanager'
 
@@ -397,7 +397,7 @@ def commandInstallPlugin():
 
             return True
 
-        valid_hash = pluginapi.get_utils().validateHash(pkobh)
+        valid_hash = stringvalidators.validate_hash(pkobh)
         real_block = False
         valid_key = stringvalidators.validate_pub_key(pkobh)
         real_key = False
@@ -485,7 +485,7 @@ def commandAddRepository():
 
         blockhash = sys.argv[2]
 
-        if pluginapi.get_utils().validateHash(blockhash):
+        if stringvalidators.validate_hash(blockhash):
             if Block.exists(blockhash):
                 try:
                     blockContent = json.loads(Block(blockhash, core = pluginapi.get_core()).getContent())
@@ -521,7 +521,7 @@ def commandRemoveRepository():
 
         blockhash = sys.argv[2]
 
-        if pluginapi.get_utils().validateHash(blockhash):
+        if stringvalidators.validate_hash(blockhash):
             if blockhash in getRepositories():
                 try:
                     removeRepository(blockhash)
diff --git a/onionr/static-data/default-plugins/pms/mailapi.py b/onionr/static-data/default-plugins/pms/mailapi.py
index af59bd93..922432a0 100755
--- a/onionr/static-data/default-plugins/pms/mailapi.py
+++ b/onionr/static-data/default-plugins/pms/mailapi.py
@@ -21,6 +21,7 @@ import sys, os, json
 from flask import Response, request, redirect, Blueprint, abort
 import core
 from onionrusers import contactmanager
+from onionrutils import stringvalidators
 sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
 import loadinbox, sentboxdb
 
@@ -34,7 +35,7 @@ def mail_ping():
 
 @flask_blueprint.route('/mail/deletemsg/<block>', methods=['POST'])
 def mail_delete(block):
-    if not c._utils.validateHash(block):
+    if not stringvalidators.validate_hash(block):
         abort(504)
     existing = kv.get('deleted_mail')
     if existing is None:
diff --git a/onionr/static-data/default-plugins/pms/main.py b/onionr/static-data/default-plugins/pms/main.py
index 532320de..dfacc915 100755
--- a/onionr/static-data/default-plugins/pms/main.py
+++ b/onionr/static-data/default-plugins/pms/main.py
@@ -23,7 +23,7 @@ import logger, config, threading, time, datetime
 from onionrblockapi import Block
 import onionrexceptions
 from onionrusers import onionrusers
-from onionrutils import stringvalidators
+from onionrutils import stringvalidators, escapeansi
 import locale, sys, os, json
 
 locale.setlocale(locale.LC_ALL, '')
@@ -148,7 +148,7 @@ class OnionrMail:
                         print('')
                     if cancel != '-q':
                         try:
-                            print(draw_border(self.myCore._utils.escapeAnsi(readBlock.bcontent.decode().strip())))
+                            print(draw_border(escapeansi.escape_ANSI(readBlock.bcontent.decode().strip())))
                         except ValueError:
                             logger.warn('Error presenting message. This is usually due to a malformed or blank message.', terminal=True)
                             pass
@@ -187,7 +187,7 @@ class OnionrMail:
                     else:
                         logger.info('Sent to: ' + self.sentMessages[self.sentboxList[int(choice)]][1], terminal=True)
                         # Print ansi escaped sent message
-                        logger.info(self.myCore._utils.escapeAnsi(self.sentMessages[self.sentboxList[int(choice)]][0]), terminal=True)
+                        logger.info(escapeansi.escape_ANSI(self.sentMessages[self.sentboxList[int(choice)]][0]), terminal=True)
                         input('Press enter to continue...')
                 finally:
                     if choice == '-q':
diff --git a/onionr/static-data/default-plugins/pms/sentboxdb.py b/onionr/static-data/default-plugins/pms/sentboxdb.py
index 7090e214..f5a56272 100755
--- a/onionr/static-data/default-plugins/pms/sentboxdb.py
+++ b/onionr/static-data/default-plugins/pms/sentboxdb.py
@@ -19,6 +19,7 @@
 '''
 import sqlite3, os
 import core
+from onionrutils import epoch
 class SentBox:
     def __init__(self, mycore):
         assert isinstance(mycore, core.Core)
@@ -60,7 +61,7 @@ class SentBox:
 
     def addToSent(self, blockID, peer, message, subject=''):
         self.connect()
-        args = (blockID, peer, message, subject, self.core._utils.getEpoch())
+        args = (blockID, peer, message, subject, epoch.get_epoch())
         self.cursor.execute('INSERT INTO sent VALUES(?, ?, ?, ?, ?)', args)
         self.conn.commit()
         self.close()
diff --git a/onionr/utils/netutils.py b/onionr/utils/netutils.py
index b6f16922..1c7bea1c 100755
--- a/onionr/utils/netutils.py
+++ b/onionr/utils/netutils.py
@@ -17,7 +17,8 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-def checkNetwork(utilsInst, torPort=0):
+from onionrutils import basicrequests
+def checkNetwork(core_inst, torPort=0):
     '''Check if we are connected to the internet (through Tor)'''
     retData = False
     connectURLs = []
@@ -26,7 +27,7 @@ def checkNetwork(utilsInst, torPort=0):
             connectURLs = connectTest.read().split(',')
 
         for url in connectURLs:
-            if utilsInst.doGetRequest(url, port=torPort, ignoreAPI=True) != False:
+            if basicrequests.do_get_request(core_inst, url, port=torPort, ignoreAPI=True) != False:
                 retData = True
                 break
     except FileNotFoundError:
diff --git a/onionr/utils/sizeutils.py b/onionr/utils/sizeutils.py
new file mode 100644
index 00000000..da121910
--- /dev/null
+++ b/onionr/utils/sizeutils.py
@@ -0,0 +1,27 @@
+import sqlite3, os
+from onionrutils import stringvalidators
+def human_size(num, suffix='B'):
+    '''
+        Converts from bytes to a human readable format.
+    '''
+    for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
+        if abs(num) < 1024.0:
+            return "%.1f %s%s" % (num, unit, suffix)
+        num /= 1024.0
+    return "%.1f %s%s" % (num, 'Yi', suffix)
+
+def size(path='.'):
+    '''
+        Returns the size of a folder's contents in bytes
+    '''
+    total = 0
+    if os.path.exists(path):
+        if os.path.isfile(path):
+            total = os.path.getsize(path)
+        else:
+            for entry in os.scandir(path):
+                if entry.is_file():
+                    total += entry.stat().st_size
+                elif entry.is_dir():
+                    total += size(entry.path)
+    return total
\ No newline at end of file

From 122eb4ee5fc4d9bde1d91ad1bfd708b531935b06 Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Tue, 25 Jun 2019 19:15:04 -0500
Subject: [PATCH 15/29] Finished removing onionrutils for the most part, except
 for some possible bugs remaining

---
 onionr/blockimporter.py                        | 4 ++--
 onionr/communicatorutils/announcenode.py       | 4 ++--
 onionr/communicatorutils/downloadblocks.py     | 4 ++--
 onionr/communicatorutils/uploadblocks.py       | 2 +-
 onionr/core.py                                 | 7 +++++--
 onionr/httpapi/miscpublicapi/announce.py       | 4 ++--
 onionr/logger.py                               | 2 +-
 onionr/onionr.py                               | 1 -
 onionr/onionrcommands/daemonlaunch.py          | 2 +-
 onionr/onionrutils/blockmetadata.py            | 1 +
 onionr/onionrutils/localcommand.py             | 5 +++--
 onionr/onionrutils/validatemetadata.py         | 4 ++--
 onionr/static-data/default-plugins/pms/main.py | 8 ++++----
 13 files changed, 26 insertions(+), 22 deletions(-)

diff --git a/onionr/blockimporter.py b/onionr/blockimporter.py
index d5e02e7d..067ec045 100755
--- a/onionr/blockimporter.py
+++ b/onionr/blockimporter.py
@@ -37,7 +37,7 @@ def importBlockFromData(content, coreInst):
 
     metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
     metadata = metas[0]
-    if validatemetadata(metadata, metas[2]): # check if metadata is valid
+    if validatemetadata.validate_metadata(coreInst, metadata, metas[2]): # check if metadata is valid
         if coreInst._crypto.verifyPow(content): # check if POW is enough/correct
             logger.info('Block passed proof, saving.', terminal=True)
             try:
@@ -46,6 +46,6 @@ def importBlockFromData(content, coreInst):
                 pass
             else:
                 coreInst.addToBlockDB(blockHash, dataSaved=True)
-                blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database
+                blockmetadata.process_block_metadata(coreInst, blockHash) # caches block metadata values to block database
                 retData = True
     return retData
\ No newline at end of file
diff --git a/onionr/communicatorutils/announcenode.py b/onionr/communicatorutils/announcenode.py
index f72010f5..38453647 100755
--- a/onionr/communicatorutils/announcenode.py
+++ b/onionr/communicatorutils/announcenode.py
@@ -20,7 +20,7 @@
 import base64
 import onionrproofs, logger
 from etc import onionrvalues
-from onionrutils import basicrequests
+from onionrutils import basicrequests, bytesconverter
 
 def announce_node(daemon):
     '''Announce our node to our peers'''
@@ -53,7 +53,7 @@ def announce_node(daemon):
             combinedNodes = ourID + peer
             if ourID != 1:
                 #TODO: Extend existingRand for i2p
-                existingRand = daemon._core._utils.bytesToStr(daemon._core.getAddressInfo(peer, 'powValue'))
+                existingRand = bytesconverter.bytes_to_str(daemon._core.getAddressInfo(peer, 'powValue'))
                 # Reset existingRand if it no longer meets the minimum POW
                 if type(existingRand) is type(None) or not existingRand.endswith('0' * ov.announce_pow):
                     existingRand = ''
diff --git a/onionr/communicatorutils/downloadblocks.py b/onionr/communicatorutils/downloadblocks.py
index ecd7b873..1a159c93 100755
--- a/onionr/communicatorutils/downloadblocks.py
+++ b/onionr/communicatorutils/downloadblocks.py
@@ -75,7 +75,7 @@ def download_blocks_from_communicator(comm_inst):
                 content = content.decode() # decode here because sha3Hash needs bytes above
                 metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
                 metadata = metas[0]
-                if validatemetadata.validate_metadata(comm_inist._core, metadata, metas[2]): # check if metadata is valid, and verify nonce
+                if validatemetadata.validate_metadata(comm_inst._core, metadata, metas[2]): # check if metadata is valid, and verify nonce
                     if comm_inst._core._crypto.verifyPow(content): # check if POW is enough/correct
                         logger.info('Attempting to save block %s...' % blockHash[:12])
                         try:
@@ -85,7 +85,7 @@ def download_blocks_from_communicator(comm_inst):
                             removeFromQueue = False
                         else:
                             comm_inst._core.addToBlockDB(blockHash, dataSaved=True)
-                            blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database
+                            blockmetadata.process_block_metadata(comm_inst._core, blockHash) # caches block metadata values to block database
                     else:
                         logger.warn('POW failed for block %s.' % blockHash)
                 else:
diff --git a/onionr/communicatorutils/uploadblocks.py b/onionr/communicatorutils/uploadblocks.py
index afe96525..0a3488a8 100755
--- a/onionr/communicatorutils/uploadblocks.py
+++ b/onionr/communicatorutils/uploadblocks.py
@@ -42,7 +42,7 @@ def upload_blocks_from_communicator(comm_inst):
                 url = 'http://' + peer + '/upload'
                 data = {'block': block.Block(bl).getRaw()}
                 proxyType = proxypicker.pick_proxy(peer)
-                logger.info("Uploading block to " + peer)
+                logger.info("Uploading block to " + peer, terminal=True)
                 if not basicrequests.do_post_request(core, url, data=data, proxyType=proxyType) == False:
                     localcommand.local_command(core, 'waitforshare/' + bl, post=True)
                     finishedUploads.append(bl)
diff --git a/onionr/core.py b/onionr/core.py
index 0d64877a..09d2cc4d 100755
--- a/onionr/core.py
+++ b/onionr/core.py
@@ -306,7 +306,7 @@ class Core:
             dateClaimed  - timestamp claimed inside the block, only as trustworthy as the block author is
             expire       - expire date for a block
         '''
-        return coredb.blockmetadb.updateblockinfo
+        return coredb.blockmetadb.updateblockinfo.update_block_info(self, hash, key, data)
 
     def insertBlock(self, data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = {}, expire=None, disableForward=False):
         '''
@@ -437,8 +437,11 @@ class Core:
             else:
                 # Tell the api server through localCommand to wait for the daemon to upload this block to make statistical analysis more difficult
                 if localcommand.local_command(self, '/ping', maxWait=10) == 'pong!':
-                    localcommand.local_command(self, '/waitforshare/' + retData, post=True, maxWait=5)
+                    if self.config.get('general.security_level', 1) > 0:
+                        localcommand.local_command(self, '/waitforshare/' + retData, post=True, maxWait=5)
                     self.daemonQueueAdd('uploadBlock', retData)
+                else:
+                    print('shite', localcommand.local_command(self, '/ping', maxWait=10))
                 self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
                 blockmetadata.process_block_metadata(self, retData)
 
diff --git a/onionr/httpapi/miscpublicapi/announce.py b/onionr/httpapi/miscpublicapi/announce.py
index f17c8d83..060bead4 100755
--- a/onionr/httpapi/miscpublicapi/announce.py
+++ b/onionr/httpapi/miscpublicapi/announce.py
@@ -21,7 +21,7 @@ import base64
 from flask import Response
 import logger
 from etc import onionrvalues
-from onionrutils import stringvalidators
+from onionrutils import stringvalidators, bytesconverter
 
 def handle_announce(clientAPI, request):
     '''
@@ -53,7 +53,7 @@ def handle_announce(clientAPI, request):
             except AttributeError:
                 pass
             if powHash.startswith('0' * onionrvalues.OnionrValues().announce_pow):
-                newNode = clientAPI._core._utils.bytesToStr(newNode)
+                newNode = bytesconverter.bytes_to_str(newNode)
                 if stringvalidators.validate_transport(newNode) and not newNode in clientAPI._core.onionrInst.communicatorInst.newPeers:
                     clientAPI._core.onionrInst.communicatorInst.newPeers.append(newNode)
                     resp = 'Success'
diff --git a/onionr/logger.py b/onionr/logger.py
index 01ada8ce..1f65d0d4 100755
--- a/onionr/logger.py
+++ b/onionr/logger.py
@@ -131,7 +131,7 @@ def raw(data, fd = sys.stdout, terminal = False):
         Outputs raw data to console without formatting
     '''
 
-    if terminal and (get_settings() & OUTPUT_TO_CONSOLE):
+    if (get_settings() & OUTPUT_TO_CONSOLE):
         try:
             ts = fd.write('%s\n' % data)
         except OSError:
diff --git a/onionr/onionr.py b/onionr/onionr.py
index 85a34020..cd16ddfe 100755
--- a/onionr/onionr.py
+++ b/onionr/onionr.py
@@ -43,7 +43,6 @@ try:
 except ImportError:
     raise Exception("You need the PySocks module (for use with socks5 proxy to use Tor)")
 
-
 class Onionr:
     def __init__(self):
         '''
diff --git a/onionr/onionrcommands/daemonlaunch.py b/onionr/onionrcommands/daemonlaunch.py
index 28202c10..c82d13de 100755
--- a/onionr/onionrcommands/daemonlaunch.py
+++ b/onionr/onionrcommands/daemonlaunch.py
@@ -60,7 +60,7 @@ def daemon(o_inst):
     logger.debug('Python version %s' % platform.python_version())
 
     if o_inst._developmentMode:
-        logger.warn('DEVELOPMENT MODE ENABLED', timestamp = False, terminal=True)
+        logger.warn('Development mode enabled', timestamp = False, terminal=True)
     net = NetController(o_inst.onionrCore.config.get('client.public.port', 59497), apiServerIP=apiHost)
     logger.info('Tor is starting...', terminal=True)
     if not net.startTor():
diff --git a/onionr/onionrutils/blockmetadata.py b/onionr/onionrutils/blockmetadata.py
index d892257f..72bdb48c 100644
--- a/onionr/onionrutils/blockmetadata.py
+++ b/onionr/onionrutils/blockmetadata.py
@@ -43,6 +43,7 @@ def process_block_metadata(core_inst, blockHash):
         myBlock.decrypt()
     if (myBlock.isEncrypted and myBlock.decrypted) or (not myBlock.isEncrypted):
         blockType = myBlock.getMetadata('type') # we would use myBlock.getType() here, but it is bugged with encrypted blocks
+        print('blockType', blockType)
         signer = bytesconverter.bytes_to_str(myBlock.signer)
         valid = myBlock.verifySig()
         if myBlock.getMetadata('newFSKey') is not None:
diff --git a/onionr/onionrutils/localcommand.py b/onionr/onionrutils/localcommand.py
index 90b0d7ac..6b5356ee 100644
--- a/onionr/onionrutils/localcommand.py
+++ b/onionr/onionrutils/localcommand.py
@@ -19,11 +19,12 @@ def local_command(core_inst, command, data='', silent = True, post=False, postDa
     if data != '':
         data = '&data=' + urllib.parse.quote_plus(data)
     payload = 'http://%s/%s%s' % (hostname, command, data)
+    print(payload)
     try:
         if post:
-            retData = requests.post(payload, data=postData, headers={'token': config.get('client.webpassword'), 'Connection':'close'}, timeout=(maxWait, maxWait)).text
+            retData = requests.post(payload, data=postData, headers={'token': core_inst.config.get('client.webpassword'), 'Connection':'close'}, timeout=(maxWait, maxWait)).text
         else:
-            retData = requests.get(payload, headers={'token': config.get('client.webpassword'), 'Connection':'close'}, timeout=(maxWait, maxWait)).text
+            retData = requests.get(payload, headers={'token': core_inst.config.get('client.webpassword'), 'Connection':'close'}, timeout=(maxWait, maxWait)).text
     except Exception as error:
         if not silent:
             logger.error('Failed to make local request (command: %s):%s' % (command, error), terminal=True)
diff --git a/onionr/onionrutils/validatemetadata.py b/onionr/onionrutils/validatemetadata.py
index 4e2e69f0..ec1f51da 100644
--- a/onionr/onionrutils/validatemetadata.py
+++ b/onionr/onionrutils/validatemetadata.py
@@ -1,7 +1,7 @@
 import json
 import logger, onionrexceptions
 from etc import onionrvalues
-from onionrutils import stringvalidators, epoch
+from onionrutils import stringvalidators, epoch, bytesconverter
 def validate_metadata(core_inst, metadata, blockData):
     '''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string'''
     # TODO, make this check sane sizes
@@ -59,7 +59,7 @@ def validate_metadata(core_inst, metadata, blockData):
         else:
             # if metadata loop gets no errors, it does not break, therefore metadata is valid
             # make sure we do not have another block with the same data content (prevent data duplication and replay attacks)
-            nonce = core_inst._utils.bytesToStr(core_inst._crypto.sha3Hash(blockData))
+            nonce = bytesconverter.bytes_to_str(core_inst._crypto.sha3Hash(blockData))
             try:
                 with open(core_inst.dataNonceFile, 'r') as nonceFile:
                     if nonce in nonceFile.read():
diff --git a/onionr/static-data/default-plugins/pms/main.py b/onionr/static-data/default-plugins/pms/main.py
index dfacc915..5c532877 100755
--- a/onionr/static-data/default-plugins/pms/main.py
+++ b/onionr/static-data/default-plugins/pms/main.py
@@ -23,7 +23,7 @@ import logger, config, threading, time, datetime
 from onionrblockapi import Block
 import onionrexceptions
 from onionrusers import onionrusers
-from onionrutils import stringvalidators, escapeansi
+from onionrutils import stringvalidators, escapeansi, bytesconverter
 import locale, sys, os, json
 
 locale.setlocale(locale.LC_ALL, '')
@@ -136,7 +136,7 @@ class OnionrMail:
                 else:
                     cancel = ''
                     readBlock.verifySig()
-                    senderDisplay = self.myCore._utils.bytesToStr(readBlock.signer)
+                    senderDisplay = bytesconverter.bytes_to_str(readBlock.signer)
                     if len(senderDisplay.strip()) == 0:
                         senderDisplay = 'Anonymous'
                     logger.info('Message received from %s' % (senderDisplay,), terminal=True)
@@ -156,7 +156,7 @@ class OnionrMail:
                             reply = logger.readline("Press enter to continue, or enter %s to reply" % ("-r",))
                             print('')
                             if reply == "-r":
-                                self.draft_message(self.myCore._utils.bytesToStr(readBlock.signer,))
+                                self.draft_message(bytesconverter.bytes_to_str(readBlock.signer,))
                         else:
                             logger.readline("Press enter to continue")
                             print('')
@@ -200,7 +200,7 @@ class OnionrMail:
         self.sentMessages = {}
         for i in self.sentboxTools.listSent():
             self.sentboxList.append(i['hash'])
-            self.sentMessages[i['hash']] = (self.myCore._utils.bytesToStr(i['message']), i['peer'], i['subject'])
+            self.sentMessages[i['hash']] = (bytesconverter.bytes_to_str(i['message']), i['peer'], i['subject'])
             if display:
                 logger.info('%s. %s - %s - (%s) - %s' % (count, i['hash'], i['peer'][:12], i['subject'], i['date']), terminal=True)
             count += 1

From a5a50a84d4e43f8cf2ce0761439a2ca46a3ef91f Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Tue, 25 Jun 2019 19:32:44 -0500
Subject: [PATCH 16/29] adjust test to use new stringvalidation module

---
 onionr/core.py                         | 129 ++++++++++++-------------
 onionr/tests/test_stringvalidations.py |   4 +-
 2 files changed, 66 insertions(+), 67 deletions(-)

diff --git a/onionr/core.py b/onionr/core.py
index 09d2cc4d..84ef5f3a 100755
--- a/onionr/core.py
+++ b/onionr/core.py
@@ -42,76 +42,75 @@ class Core:
         if not self.dataDir.endswith('/'):
             self.dataDir += '/'
 
-        #try:
-        self.usageFile = self.dataDir + 'disk-usage.txt'
-        self.config = config
-        self.maxBlockSize = 10000000 # max block size in bytes
+        try:
+            self.usageFile = self.dataDir + 'disk-usage.txt'
+            self.config = config
+            self.maxBlockSize = 10000000 # max block size in bytes
 
-        self.onionrInst = None
-        self.queueDB = self.dataDir + 'queue.db'
-        self.peerDB = self.dataDir + 'peers.db'
-        self.blockDB = self.dataDir + 'blocks.db'
-        self.blockDataLocation = self.dataDir + 'blocks/'
-        self.blockDataDB = self.blockDataLocation + 'block-data.db'
-        self.publicApiHostFile = self.dataDir + 'public-host.txt'
-        self.privateApiHostFile = self.dataDir + 'private-host.txt'
-        self.addressDB = self.dataDir + 'address.db'
-        self.hsAddress = ''
-        self.i2pAddress = config.get('i2p.own_addr', None)
-        self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt'
-        self.bootstrapList = []
-        self.requirements = onionrvalues.OnionrValues()
-        self.torPort = torPort
-        self.dataNonceFile = self.dataDir + 'block-nonces.dat'
-        self.dbCreate = dbcreator.DBCreator(self)
-        self.forwardKeysFile = self.dataDir + 'forward-keys.db'
-        self.keyStore = simplekv.DeadSimpleKV(self.dataDir + 'cachedstorage.dat', refresh_seconds=5)
-        self.storage_counter = storagecounter.StorageCounter(self)
-        
-        # Socket data, defined here because of multithreading constraints with gevent
-        self.killSockets = False
-        self.startSocket = {}
-        self.socketServerConnData = {}
-        self.socketReasons = {}
-        self.socketServerResponseData = {}
+            self.onionrInst = None
+            self.queueDB = self.dataDir + 'queue.db'
+            self.peerDB = self.dataDir + 'peers.db'
+            self.blockDB = self.dataDir + 'blocks.db'
+            self.blockDataLocation = self.dataDir + 'blocks/'
+            self.blockDataDB = self.blockDataLocation + 'block-data.db'
+            self.publicApiHostFile = self.dataDir + 'public-host.txt'
+            self.privateApiHostFile = self.dataDir + 'private-host.txt'
+            self.addressDB = self.dataDir + 'address.db'
+            self.hsAddress = ''
+            self.i2pAddress = config.get('i2p.own_addr', None)
+            self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt'
+            self.bootstrapList = []
+            self.requirements = onionrvalues.OnionrValues()
+            self.torPort = torPort
+            self.dataNonceFile = self.dataDir + 'block-nonces.dat'
+            self.dbCreate = dbcreator.DBCreator(self)
+            self.forwardKeysFile = self.dataDir + 'forward-keys.db'
+            self.keyStore = simplekv.DeadSimpleKV(self.dataDir + 'cachedstorage.dat', refresh_seconds=5)
+            self.storage_counter = storagecounter.StorageCounter(self)
+            
+            # Socket data, defined here because of multithreading constraints with gevent
+            self.killSockets = False
+            self.startSocket = {}
+            self.socketServerConnData = {}
+            self.socketReasons = {}
+            self.socketServerResponseData = {}
 
-        if not os.path.exists(self.dataDir):
-            os.mkdir(self.dataDir)
-        if not os.path.exists(self.dataDir + 'blocks/'):
-            os.mkdir(self.dataDir + 'blocks/')
-        if not os.path.exists(self.blockDB):
-            self.createBlockDB()
-        if not os.path.exists(self.forwardKeysFile):
-            self.dbCreate.createForwardKeyDB()
-        if not os.path.exists(self.peerDB):
-            self.createPeerDB()
-        if not os.path.exists(self.addressDB):
-            self.createAddressDB()
+            if not os.path.exists(self.dataDir):
+                os.mkdir(self.dataDir)
+            if not os.path.exists(self.dataDir + 'blocks/'):
+                os.mkdir(self.dataDir + 'blocks/')
+            if not os.path.exists(self.blockDB):
+                self.createBlockDB()
+            if not os.path.exists(self.forwardKeysFile):
+                self.dbCreate.createForwardKeyDB()
+            if not os.path.exists(self.peerDB):
+                self.createPeerDB()
+            if not os.path.exists(self.addressDB):
+                self.createAddressDB()
 
-        if os.path.exists(self.dataDir + '/hs/hostname'):
-            with open(self.dataDir + '/hs/hostname', 'r') as hs:
-                self.hsAddress = hs.read().strip()
+            if os.path.exists(self.dataDir + '/hs/hostname'):
+                with open(self.dataDir + '/hs/hostname', 'r') as hs:
+                    self.hsAddress = hs.read().strip()
 
-        # Load bootstrap address list
-        if os.path.exists(self.bootstrapFileLocation):
-            with open(self.bootstrapFileLocation, 'r') as bootstrap:
-                bootstrap = bootstrap.read()
-            for i in bootstrap.split('\n'):
-                self.bootstrapList.append(i)
-        else:
-            logger.warn('Warning: address bootstrap file not found ' + self.bootstrapFileLocation)
+            # Load bootstrap address list
+            if os.path.exists(self.bootstrapFileLocation):
+                with open(self.bootstrapFileLocation, 'r') as bootstrap:
+                    bootstrap = bootstrap.read()
+                for i in bootstrap.split('\n'):
+                    self.bootstrapList.append(i)
+            else:
+                logger.warn('Warning: address bootstrap file not found ' + self.bootstrapFileLocation)
 
-        self.use_subprocess = powchoice.use_subprocess(self)
-        # Initialize the crypto object
-        self._crypto = onionrcrypto.OnionrCrypto(self)
-        self._blacklist = onionrblacklist.OnionrBlackList(self)
-        self.serializer = serializeddata.SerializedData(self)
+            self.use_subprocess = powchoice.use_subprocess(self)
+            # Initialize the crypto object
+            self._crypto = onionrcrypto.OnionrCrypto(self)
+            self._blacklist = onionrblacklist.OnionrBlackList(self)
+            self.serializer = serializeddata.SerializedData(self)
 
-        # except Exception as error:
-        #     print(str(error))
-        #     logger.error('Failed to initialize core Onionr library.', error=error, terminal=True)
-        #     logger.fatal('Cannot recover from error.', terminal=True)
-        #     sys.exit(1)
+        except Exception as error:
+            logger.error('Failed to initialize core Onionr library.', error=error, terminal=True)
+            logger.fatal('Cannot recover from error.', terminal=True)
+            sys.exit(1)
         return
 
     def refreshFirstStartVars(self):
@@ -441,7 +440,7 @@ class Core:
                         localcommand.local_command(self, '/waitforshare/' + retData, post=True, maxWait=5)
                     self.daemonQueueAdd('uploadBlock', retData)
                 else:
-                    print('shite', localcommand.local_command(self, '/ping', maxWait=10))
+                    pass
                 self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
                 blockmetadata.process_block_metadata(self, retData)
 
diff --git a/onionr/tests/test_stringvalidations.py b/onionr/tests/test_stringvalidations.py
index 4eb7f113..72d5fb20 100755
--- a/onionr/tests/test_stringvalidations.py
+++ b/onionr/tests/test_stringvalidations.py
@@ -35,11 +35,11 @@ class OnionrValidations(unittest.TestCase):
 
         for valid in valids:
             print('testing', valid)
-            self.assertTrue(c._utils.validatePubKey(valid))
+            self.assertTrue(stringvalidators.validate_pub_key(valid))
 
         for x in invalid:
             #print('testing', x)
-            self.assertFalse(c._utils.validatePubKey(x))
+            self.assertFalse(stringvalidators.validate_pub_key(x))
     
     def test_integer_string(self):
         valid = ["1", "100", 100, "-5", -5]

From 0b34951423952ebfe7dce8e2f1d0dcd5585c7c76 Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Tue, 25 Jun 2019 19:33:55 -0500
Subject: [PATCH 17/29] redo new logging

---
 onionr/logger.py                   | 2 +-
 onionr/onionrutils/localcommand.py | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/onionr/logger.py b/onionr/logger.py
index 1f65d0d4..01ada8ce 100755
--- a/onionr/logger.py
+++ b/onionr/logger.py
@@ -131,7 +131,7 @@ def raw(data, fd = sys.stdout, terminal = False):
         Outputs raw data to console without formatting
     '''
 
-    if (get_settings() & OUTPUT_TO_CONSOLE):
+    if terminal and (get_settings() & OUTPUT_TO_CONSOLE):
         try:
             ts = fd.write('%s\n' % data)
         except OSError:
diff --git a/onionr/onionrutils/localcommand.py b/onionr/onionrutils/localcommand.py
index 6b5356ee..c3bf3ceb 100644
--- a/onionr/onionrutils/localcommand.py
+++ b/onionr/onionrutils/localcommand.py
@@ -19,7 +19,6 @@ def local_command(core_inst, command, data='', silent = True, post=False, postDa
     if data != '':
         data = '&data=' + urllib.parse.quote_plus(data)
     payload = 'http://%s/%s%s' % (hostname, command, data)
-    print(payload)
     try:
         if post:
             retData = requests.post(payload, data=postData, headers={'token': core_inst.config.get('client.webpassword'), 'Connection':'close'}, timeout=(maxWait, maxWait)).text

From c8a50c5956daae07233dcd2ca42557636e4c6dad Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Tue, 25 Jun 2019 23:48:24 -0500
Subject: [PATCH 18/29] fixed broken waitforshare

---
 README.md                           | 2 +-
 onionr/core.py                      | 6 +++---
 onionr/onionrutils/blockmetadata.py | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 4417e31f..0785e78b 100644
--- a/README.md
+++ b/README.md
@@ -123,7 +123,7 @@ Note: probably not tax deductible
 
 Email: beardog [ at ] mailbox.org
 
-Onionr Mail: TRH763JURNY47QPBTTQ4LLPYCYQK6Q5YA33R6GANKZK5C5DKCIGQ====
+Onionr Mail: TRH763JURNY47QPBTTQ4LLPYCYQK6Q5YA33R6GANKZK5C5DKCIGQ
 
 ## Disclaimers and legal
 
diff --git a/onionr/core.py b/onionr/core.py
index 84ef5f3a..195a8d04 100755
--- a/onionr/core.py
+++ b/onionr/core.py
@@ -17,12 +17,12 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import os, sys, time, json, uuid
+import os, sys, json
 import logger, netcontroller, config
 from onionrblockapi import Block
 import coredb
 import deadsimplekv as simplekv
-import onionrutils, onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions
+import onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions
 import onionrblacklist
 from onionrusers import onionrusers
 from onionrstorage import removeblock, setdata
@@ -436,7 +436,7 @@ class Core:
             else:
                 # Tell the api server through localCommand to wait for the daemon to upload this block to make statistical analysis more difficult
                 if localcommand.local_command(self, '/ping', maxWait=10) == 'pong!':
-                    if self.config.get('general.security_level', 1) > 0:
+                    if self.config.get('general.security_level', 1) == 0:
                         localcommand.local_command(self, '/waitforshare/' + retData, post=True, maxWait=5)
                     self.daemonQueueAdd('uploadBlock', retData)
                 else:
diff --git a/onionr/onionrutils/blockmetadata.py b/onionr/onionrutils/blockmetadata.py
index 72bdb48c..91fa1952 100644
--- a/onionr/onionrutils/blockmetadata.py
+++ b/onionr/onionrutils/blockmetadata.py
@@ -43,7 +43,7 @@ def process_block_metadata(core_inst, blockHash):
         myBlock.decrypt()
     if (myBlock.isEncrypted and myBlock.decrypted) or (not myBlock.isEncrypted):
         blockType = myBlock.getMetadata('type') # we would use myBlock.getType() here, but it is bugged with encrypted blocks
-        print('blockType', blockType)
+
         signer = bytesconverter.bytes_to_str(myBlock.signer)
         valid = myBlock.verifySig()
         if myBlock.getMetadata('newFSKey') is not None:

From 3bb0e48a06ed71b8baa03ed5f795f36bc31e24d4 Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Wed, 26 Jun 2019 00:26:50 -0500
Subject: [PATCH 19/29] added some of the missing boilerplate

---
 README.md                                    |  2 +-
 onionr/coredb/blockmetadb/__init__.py        | 19 +++++++++++++++++++
 onionr/coredb/blockmetadb/add.py             | 19 +++++++++++++++++++
 onionr/coredb/blockmetadb/expiredblocks.py   | 19 +++++++++++++++++++
 onionr/coredb/blockmetadb/updateblockinfo.py | 20 ++++++++++++++++++++
 onionr/coredb/daemonqueue/__init__.py        | 20 ++++++++++++++++++++
 6 files changed, 98 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 0785e78b..ee96efd4 100644
--- a/README.md
+++ b/README.md
@@ -11,7 +11,7 @@
 (***pre-alpha & experimental, not well tested or easy to use yet***)
 
 [![Open Source Love](https://badges.frapsoft.com/os/v3/open-source.png?v=103)](https://github.com/ellerbrock/open-source-badges/)
-<img src='https://gitlab.com/beardog/Onionr/badges/master/build.svg'> - [Onionr.net](https://onionr.net/)
+<img src='https://gitlab.com/beardog/Onionr/badges/master/build.svg'> - [Onionr.net](https://onionr.net/) - [.onion](http://onionr.onionkvc5ibm37bmxwr56bdxcdnb6w3wm4bdghh5qo6f6za7gn7styid.onion/)
 
 <hr>
 
diff --git a/onionr/coredb/blockmetadb/__init__.py b/onionr/coredb/blockmetadb/__init__.py
index 9aa991a5..409696dc 100644
--- a/onionr/coredb/blockmetadb/__init__.py
+++ b/onionr/coredb/blockmetadb/__init__.py
@@ -1,3 +1,22 @@
+'''
+    Onionr - Private P2P Communication
+
+    This module works with information relating to blocks stored on the node
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
 import sqlite3
 from . import expiredblocks, updateblockinfo, add
 def get_block_list(core_inst, dateRec = None, unsaved = False):
diff --git a/onionr/coredb/blockmetadb/add.py b/onionr/coredb/blockmetadb/add.py
index bdf9457f..cfd8305c 100644
--- a/onionr/coredb/blockmetadb/add.py
+++ b/onionr/coredb/blockmetadb/add.py
@@ -1,3 +1,22 @@
+'''
+    Onionr - Private P2P Communication
+
+    Add an entry to the block metadata database
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
 import os, sqlite3
 from onionrutils import epoch, blockmetadata
 def add_to_block_DB(core_inst, newHash, selfInsert=False, dataSaved=False):
diff --git a/onionr/coredb/blockmetadb/expiredblocks.py b/onionr/coredb/blockmetadb/expiredblocks.py
index 0b102fe6..9e01d3ac 100644
--- a/onionr/coredb/blockmetadb/expiredblocks.py
+++ b/onionr/coredb/blockmetadb/expiredblocks.py
@@ -1,3 +1,22 @@
+'''
+    Onionr - Private P2P Communication
+
+    Get a list of expired blocks still stored
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
 import sqlite3
 from onionrutils import epoch
 def get_expired_blocks(core_inst):
diff --git a/onionr/coredb/blockmetadb/updateblockinfo.py b/onionr/coredb/blockmetadb/updateblockinfo.py
index 05f0fc8a..bbb5d969 100644
--- a/onionr/coredb/blockmetadb/updateblockinfo.py
+++ b/onionr/coredb/blockmetadb/updateblockinfo.py
@@ -1,3 +1,23 @@
+'''
+    Onionr - Private P2P Communication
+
+    Update block information in the metadata database by a field name
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
 import sqlite3
 def update_block_info(core_inst, hash, key, data):
     if key not in ('dateReceived', 'decrypted', 'dataType', 'dataFound', 'dataSaved', 'sig', 'author', 'dateClaimed', 'expire'):
diff --git a/onionr/coredb/daemonqueue/__init__.py b/onionr/coredb/daemonqueue/__init__.py
index 252cbc37..b494a45a 100644
--- a/onionr/coredb/daemonqueue/__init__.py
+++ b/onionr/coredb/daemonqueue/__init__.py
@@ -1,3 +1,23 @@
+'''
+    Onionr - Private P2P Communication
+
+    Write and read the daemon queue, which is how messages are passed into the onionr daemon in a more
+    direct way than the http api
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
 import sqlite3, os
 import onionrevents as events
 from onionrutils import localcommand, epoch

From 60d47e1e418350615b6d600e80d7e07c2bdc2142 Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Wed, 26 Jun 2019 00:41:49 -0500
Subject: [PATCH 20/29] remove utils from netcheck

---
 onionr/communicatorutils/netcheck.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/onionr/communicatorutils/netcheck.py b/onionr/communicatorutils/netcheck.py
index 85d10605..81369d58 100755
--- a/onionr/communicatorutils/netcheck.py
+++ b/onionr/communicatorutils/netcheck.py
@@ -32,7 +32,7 @@ def net_check(comm_inst):
                 rec = True
         except ValueError:
             pass
-        if not rec and not netutils.checkNetwork(c._utils, torPort=comm_inst.proxyPort):
+        if not rec and not netutils.checkNetwork(c, torPort=comm_inst.proxyPort):
             if not comm_inst.shutdown:
                 logger.warn('Network check failed, are you connected to the Internet, and is Tor working?')
             comm_inst.isOnline = False

From 7f4605998c4478d9d128949bb9b5670fa278b523 Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Wed, 26 Jun 2019 01:34:34 -0500
Subject: [PATCH 21/29] fix some commands terminal output

---
 onionr/coredb/keydb/addkeys.py       | 19 +++++++++++++++++++
 onionr/coredb/keydb/listkeys.py      | 19 +++++++++++++++++++
 onionr/coredb/keydb/removekeys.py    | 19 +++++++++++++++++++
 onionr/coredb/keydb/transportinfo.py | 19 +++++++++++++++++++
 onionr/onionr.py                     | 10 +++++-----
 5 files changed, 81 insertions(+), 5 deletions(-)

diff --git a/onionr/coredb/keydb/addkeys.py b/onionr/coredb/keydb/addkeys.py
index 20d7500a..ca352d17 100644
--- a/onionr/coredb/keydb/addkeys.py
+++ b/onionr/coredb/keydb/addkeys.py
@@ -1,3 +1,22 @@
+'''
+    Onionr - Private P2P Communication
+
+    add user keys or transport addresses
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
 import sqlite3
 import onionrevents as events
 from onionrutils import stringvalidators
diff --git a/onionr/coredb/keydb/listkeys.py b/onionr/coredb/keydb/listkeys.py
index 3ba23678..50aff637 100644
--- a/onionr/coredb/keydb/listkeys.py
+++ b/onionr/coredb/keydb/listkeys.py
@@ -1,3 +1,22 @@
+'''
+    Onionr - Private P2P Communication
+
+    get lists for user keys or transport addresses
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
 import sqlite3
 import logger
 from onionrutils import epoch
diff --git a/onionr/coredb/keydb/removekeys.py b/onionr/coredb/keydb/removekeys.py
index ce4ba5fc..c0f1d7a6 100644
--- a/onionr/coredb/keydb/removekeys.py
+++ b/onionr/coredb/keydb/removekeys.py
@@ -1,3 +1,22 @@
+'''
+    Onionr - Private P2P Communication
+
+    Remove a transport address but don't ban them
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
 import sqlite3
 import onionrevents as events
 from onionrutils import stringvalidators
diff --git a/onionr/coredb/keydb/transportinfo.py b/onionr/coredb/keydb/transportinfo.py
index 9fd642ec..2fcd873b 100644
--- a/onionr/coredb/keydb/transportinfo.py
+++ b/onionr/coredb/keydb/transportinfo.py
@@ -1,3 +1,22 @@
+'''
+    Onionr - Private P2P Communication
+
+    get or set transport address meta information
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
 import sqlite3
 def get_address_info(core_inst, address, info):
     '''
diff --git a/onionr/onionr.py b/onionr/onionr.py
index cd16ddfe..fe06457d 100755
--- a/onionr/onionr.py
+++ b/onionr/onionr.py
@@ -261,13 +261,13 @@ class Onionr:
         if len(sys.argv) >= 4:
             config.reload()
             config.set(sys.argv[2], sys.argv[3], True)
-            logger.debug('Configuration file updated.')
+            logger.info('Configuration file updated.', terminal=True)
         elif len(sys.argv) >= 3:
             config.reload()
-            logger.info(logger.colors.bold + sys.argv[2] + ': ' + logger.colors.reset + str(config.get(sys.argv[2], logger.colors.fg.red + 'Not set.')))
+            logger.info(logger.colors.bold + sys.argv[2] + ': ' + logger.colors.reset + str(config.get(sys.argv[2], logger.colors.fg.red + 'Not set.')), terminal=True)
         else:
-            logger.info(logger.colors.bold + 'Get a value: ' + logger.colors.reset + sys.argv[0] + ' ' + sys.argv[1] + ' <key>')
-            logger.info(logger.colors.bold + 'Set a value: ' + logger.colors.reset + sys.argv[0] + ' ' + sys.argv[1] + ' <key> <value>')
+            logger.info(logger.colors.bold + 'Get a value: ' + logger.colors.reset + sys.argv[0] + ' ' + sys.argv[1] + ' <key>', terminal=True)
+            logger.info(logger.colors.bold + 'Set a value: ' + logger.colors.reset + sys.argv[0] + ' ' + sys.argv[1] + ' <key> <value>', terminal=True)
 
     def execute(self, argument):
         '''
@@ -302,7 +302,7 @@ class Onionr:
         '''
             Displays a list of keys (used to be called peers) (?)
         '''
-        logger.info('%sPublic keys in database: \n%s%s' % (logger.colors.fg.lightgreen, logger.colors.fg.green, '\n'.join(self.onionrCore.listPeers())))
+        logger.info('%sPublic keys in database: \n%s%s' % (logger.colors.fg.lightgreen, logger.colors.fg.green, '\n'.join(self.onionrCore.listPeers())), terminal=True)
 
     def addPeer(self):
         '''

From 94ba3e29d1e119d39bdc2fd6202042905aa2cc9f Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Wed, 26 Jun 2019 14:54:13 -0500
Subject: [PATCH 22/29] do not show traceback when adding existing determiistic
 key

---
 onionr/onionrcommands/pubkeymanager.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/onionr/onionrcommands/pubkeymanager.py b/onionr/onionrcommands/pubkeymanager.py
index a6b067bc..72db422a 100755
--- a/onionr/onionrcommands/pubkeymanager.py
+++ b/onionr/onionrcommands/pubkeymanager.py
@@ -20,7 +20,7 @@
 
 import sys, getpass
 import logger, onionrexceptions
-from onionrutils import stringvalidators
+from onionrutils import stringvalidators, bytesconverter
 from onionrusers import onionrusers, contactmanager
 import unpaddedbase32
 def add_ID(o_inst):
@@ -45,9 +45,13 @@ def add_ID(o_inst):
         else:
             logger.error('Passwords do not match.', terminal=True)
             sys.exit(1)
-        o_inst.onionrCore._crypto.keyManager.addKey(pubKey=newID, 
-        privKey=privKey)
-    logger.info('Added ID: %s' % (o_inst.onionrUtils.bytesToStr(newID),), terminal=True)
+        try:
+            o_inst.onionrCore._crypto.keyManager.addKey(pubKey=newID, 
+            privKey=privKey)
+        except ValueError:
+            logger.error('That ID is already available, you can change to it with the change-id command.', terminal=True)
+            return
+    logger.info('Added ID: %s' % (bytesconverter.bytes_to_str(newID),), terminal=True)
 
 def change_ID(o_inst):
     try:

From 683aaa39782ffe297b12e6203a3063143d4a51b3 Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Wed, 26 Jun 2019 15:13:36 -0500
Subject: [PATCH 23/29] added boilerplates

---
 onionr/coredb/keydb/userinfo.py          | 19 +++++++++++++++++++
 onionr/httpapi/__init__.py               |  2 +-
 onionr/onionrutils/basicrequests.py      | 19 +++++++++++++++++++
 onionr/onionrutils/blockmetadata.py      | 19 +++++++++++++++++++
 onionr/onionrutils/bytesconverter.py     |  5 +++--
 onionr/onionrutils/checkcommunicator.py  | 19 +++++++++++++++++++
 onionr/onionrutils/epoch.py              | 19 +++++++++++++++++++
 onionr/onionrutils/escapeansi.py         |  2 +-
 onionr/onionrutils/getclientapiserver.py | 19 +++++++++++++++++++
 onionr/onionrutils/importnewblocks.py    | 19 +++++++++++++++++++
 onionr/onionrutils/localcommand.py       | 19 +++++++++++++++++++
 onionr/onionrutils/mnemonickeys.py       | 19 +++++++++++++++++++
 onionr/onionrutils/stringvalidators.py   | 19 +++++++++++++++++++
 onionr/onionrutils/validatemetadata.py   | 19 +++++++++++++++++++
 14 files changed, 214 insertions(+), 4 deletions(-)

diff --git a/onionr/coredb/keydb/userinfo.py b/onionr/coredb/keydb/userinfo.py
index 84a737d6..b7a43fd4 100644
--- a/onionr/coredb/keydb/userinfo.py
+++ b/onionr/coredb/keydb/userinfo.py
@@ -1,3 +1,22 @@
+'''
+    Onionr - Private P2P Communication
+
+    get or set information about a user id
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
 import sqlite3
 def get_user_info(core_inst, peer, info):
     '''
diff --git a/onionr/httpapi/__init__.py b/onionr/httpapi/__init__.py
index f38d80bd..b3d6c3b3 100755
--- a/onionr/httpapi/__init__.py
+++ b/onionr/httpapi/__init__.py
@@ -1,5 +1,5 @@
 '''
-    Onionr - P2P Anonymous Storage Network
+    Onionr - Private P2P Communication
 
     This file registers plugin's flask blueprints for the client http server
 '''
diff --git a/onionr/onionrutils/basicrequests.py b/onionr/onionrutils/basicrequests.py
index 1435db28..1595b433 100644
--- a/onionr/onionrutils/basicrequests.py
+++ b/onionr/onionrutils/basicrequests.py
@@ -1,3 +1,22 @@
+'''
+    Onionr - Private P2P Communication
+
+    Do HTTP GET or POST requests through a proxy
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
 import requests
 import logger, onionrexceptions
 def do_post_request(core_inst, url, data={}, port=0, proxyType='tor'):
diff --git a/onionr/onionrutils/blockmetadata.py b/onionr/onionrutils/blockmetadata.py
index 91fa1952..7b789edf 100644
--- a/onionr/onionrutils/blockmetadata.py
+++ b/onionr/onionrutils/blockmetadata.py
@@ -1,3 +1,22 @@
+'''
+    Onionr - Private P2P Communication
+
+    Module to fetch block metadata from raw block data and process it
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
 import json, sqlite3
 import logger, onionrevents
 from onionrusers import onionrusers
diff --git a/onionr/onionrutils/bytesconverter.py b/onionr/onionrutils/bytesconverter.py
index cf69a91d..5df4d673 100644
--- a/onionr/onionrutils/bytesconverter.py
+++ b/onionr/onionrutils/bytesconverter.py
@@ -1,13 +1,14 @@
 def str_to_bytes(data):
+    '''Converts a string to bytes with .encode()'''
     try:
-        data = data.encode()
+        data = data.encode('UTF-8')
     except AttributeError:
         pass
     return data
 
 def bytes_to_str(data):
     try:
-        data = data.decode()
+        data = data.decode('UTF-8')
     except AttributeError:
         pass
     return data
\ No newline at end of file
diff --git a/onionr/onionrutils/checkcommunicator.py b/onionr/onionrutils/checkcommunicator.py
index 07f8dc1a..cfc2c31a 100644
--- a/onionr/onionrutils/checkcommunicator.py
+++ b/onionr/onionrutils/checkcommunicator.py
@@ -1,3 +1,22 @@
+'''
+    Onionr - Private P2P Communication
+
+    Check if the communicator is running
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
 import time, os
 def is_communicator_running(core_inst, timeout = 5, interval = 0.1):
     try:
diff --git a/onionr/onionrutils/epoch.py b/onionr/onionrutils/epoch.py
index 0cda05cb..01922a4c 100644
--- a/onionr/onionrutils/epoch.py
+++ b/onionr/onionrutils/epoch.py
@@ -1,3 +1,22 @@
+'''
+    Onionr - Private P2P Communication
+
+    Get floored epoch, or rounded epoch
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
 import math, time
 def get_rounded_epoch(roundS=60):
     '''
diff --git a/onionr/onionrutils/escapeansi.py b/onionr/onionrutils/escapeansi.py
index 1cd5862a..1f88e149 100644
--- a/onionr/onionrutils/escapeansi.py
+++ b/onionr/onionrutils/escapeansi.py
@@ -3,7 +3,7 @@ def escape_ANSI(line):
     '''
         Remove ANSI escape codes from a string with regex
 
-        taken or adapted from: https://stackoverflow.com/a/38662876 by user https://stackoverflow.com/users/802365/%c3%89douard-lopez
+        adapted from: https://stackoverflow.com/a/38662876 by user https://stackoverflow.com/users/802365/%c3%89douard-lopez
         cc-by-sa-3 license https://creativecommons.org/licenses/by-sa/3.0/
     '''
     ansi_escape = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -/]*[@-~]')
diff --git a/onionr/onionrutils/getclientapiserver.py b/onionr/onionrutils/getclientapiserver.py
index 23020f66..e0afe61b 100644
--- a/onionr/onionrutils/getclientapiserver.py
+++ b/onionr/onionrutils/getclientapiserver.py
@@ -1,3 +1,22 @@
+'''
+    Onionr - Private P2P Communication
+
+    Return the client api server address and port, which is usually random
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
 def get_client_API_server(core_inst):
     retData = ''
     try:
diff --git a/onionr/onionrutils/importnewblocks.py b/onionr/onionrutils/importnewblocks.py
index bfd985a2..6accd947 100644
--- a/onionr/onionrutils/importnewblocks.py
+++ b/onionr/onionrutils/importnewblocks.py
@@ -1,3 +1,22 @@
+'''
+    Onionr - Private P2P Communication
+
+    import new blocks from disk, providing transport agnosticism
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
 import glob
 import logger, core
 from onionrutils import blockmetadata
diff --git a/onionr/onionrutils/localcommand.py b/onionr/onionrutils/localcommand.py
index c3bf3ceb..13495a3c 100644
--- a/onionr/onionrutils/localcommand.py
+++ b/onionr/onionrutils/localcommand.py
@@ -1,3 +1,22 @@
+'''
+    Onionr - Private P2P Communication
+
+    send a command to the local API server
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
 import urllib, requests, time
 import logger
 from onionrutils import getclientapiserver
diff --git a/onionr/onionrutils/mnemonickeys.py b/onionr/onionrutils/mnemonickeys.py
index 30c7b1c4..56085536 100644
--- a/onionr/onionrutils/mnemonickeys.py
+++ b/onionr/onionrutils/mnemonickeys.py
@@ -1,3 +1,22 @@
+'''
+    Onionr - Private P2P Communication
+
+    convert a base32 string (intended for ed25519 user ids) to pgp word list
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
 import base64
 from etc import pgpwords
 def get_human_readable_ID(core_inst, pub=''):
diff --git a/onionr/onionrutils/stringvalidators.py b/onionr/onionrutils/stringvalidators.py
index 87fa4e8f..5b7fbf6f 100644
--- a/onionr/onionrutils/stringvalidators.py
+++ b/onionr/onionrutils/stringvalidators.py
@@ -1,3 +1,22 @@
+'''
+    Onionr - Private P2P Communication
+
+    validate various string data types
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
 import base64, string
 import unpaddedbase32, nacl.signing, nacl.encoding
 from onionrutils import bytesconverter
diff --git a/onionr/onionrutils/validatemetadata.py b/onionr/onionrutils/validatemetadata.py
index ec1f51da..2800c24a 100644
--- a/onionr/onionrutils/validatemetadata.py
+++ b/onionr/onionrutils/validatemetadata.py
@@ -1,3 +1,22 @@
+'''
+    Onionr - Private P2P Communication
+
+    validate new block's metadata
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
 import json
 import logger, onionrexceptions
 from etc import onionrvalues

From 44bcbac8ef27caec9b1ad175326b11b0c21f5f41 Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Thu, 27 Jun 2019 01:36:52 -0500
Subject: [PATCH 24/29] + added optimization check * fixed stats table * added
 main docs file (WIP)

---
 README.md                                 |  2 +-
 docs/README.md                            |  3 +++
 onionr/onionr.py                          |  8 ++++++-
 onionr/onionrcommands/onionrstatistics.py | 11 ++++-----
 onionr/utils/detectoptimization.py        | 27 +++++++++++++++++++++++
 5 files changed, 44 insertions(+), 7 deletions(-)
 create mode 100644 docs/README.md
 create mode 100755 onionr/utils/detectoptimization.py

diff --git a/README.md b/README.md
index ee96efd4..024b6118 100644
--- a/README.md
+++ b/README.md
@@ -19,7 +19,7 @@
 
 # About
 
-Onionr is a decentralized, peer-to-peer communication and storage network, designed to be anonymous and resistant to (meta)data analysis, spam, and corruption.
+Onionr is a decentralized, peer-to-peer communication network, designed to be anonymous and resistant to (meta)data analysis, spam, and corruption.
 
 Onionr stores data in independent packages referred to as 'blocks'. The blocks are synced to all other nodes in the network. Blocks and user IDs cannot be easily proven to have been created by a particular user. Even if there is enough evidence to believe that a specific user created a block, nodes still operate behind Tor or I2P and as such cannot be trivially unmasked.
 
diff --git a/docs/README.md b/docs/README.md
new file mode 100644
index 00000000..343c6c17
--- /dev/null
+++ b/docs/README.md
@@ -0,0 +1,3 @@
+# Onionr Documentation
+
+The Onionr [whitepaper](whitepaper.md) is the best place to start both for users and developers.
diff --git a/onionr/onionr.py b/onionr/onionr.py
index fe06457d..befbf439 100755
--- a/onionr/onionr.py
+++ b/onionr/onionr.py
@@ -27,8 +27,14 @@ ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION
 API_VERSION = '0' # increments of 1; only change when something fundamental about how the API works changes. This way other nodes know how to communicate without learning too much information about you.
 MIN_PY_VERSION = 6
 if sys.version_info[0] == 2 or sys.version_info[1] < MIN_PY_VERSION:
-    sys.stderr.write('Error, Onionr requires Python 3.%s+' % (MIN_PY_VERSION,))
+    sys.stderr.write('Error, Onionr requires Python 3.%s+\n' % (MIN_PY_VERSION,))
     sys.exit(1)
+
+from utils import detectoptimization
+if detectoptimization.detect_optimization():
+    sys.stderr.write('Error, Onionr cannot be run in optimized mode\n')
+    sys.exit(1)
+
 import os, base64, random, shutil, time, platform, signal
 from threading import Thread
 import api, core, config, logger, onionrplugins as plugins, onionrevents as events
diff --git a/onionr/onionrcommands/onionrstatistics.py b/onionr/onionrcommands/onionrstatistics.py
index 83d62baa..c01c5e1b 100755
--- a/onionr/onionrcommands/onionrstatistics.py
+++ b/onionr/onionrcommands/onionrstatistics.py
@@ -18,10 +18,11 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 import os, uuid, time
-import logger, onionrutils
+import logger
 from onionrblockapi import Block
 import onionr
 from onionrutils import checkcommunicator, mnemonickeys
+from utils import sizeutils
 
 def show_stats(o_inst):
     try:
@@ -34,9 +35,9 @@ def show_stats(o_inst):
 
             # file and folder size stats
             'div1' : True, # this creates a solid line across the screen, a div
-            'Total Block Size' : onionrutils.humanSize(onionrutils.size(o_inst.dataDir + 'blocks/')),
-            'Total Plugin Size' : onionrutils.humanSize(onionrutils.size(o_inst.dataDir + 'plugins/')),
-            'Log File Size' : onionrutils.humanSize(onionrutils.size(o_inst.dataDir + 'output.log')),
+            'Total Block Size' : sizeutils.human_size(sizeutils.size(o_inst.dataDir + 'blocks/')),
+            'Total Plugin Size' : sizeutils.human_size(sizeutils.size(o_inst.dataDir + 'plugins/')),
+            'Log File Size' : sizeutils.human_size(sizeutils.size(o_inst.dataDir + 'output.log')),
 
             # count stats
             'div2' : True,
@@ -80,7 +81,7 @@ def show_stats(o_inst):
                 logger.info(colors['border'] + '-' * (maxlength + 1) + '+' + colors['reset'], terminal=True)
         logger.info(colors['border'] + '-' * (maxlength + 1) + '+' + colors['reset'], terminal=True)
     except Exception as e:
-        logger.error('Failed to generate statistics table.', error = e, timestamp = False, terminal=True)
+        logger.error('Failed to generate statistics table. ' + str(e), error = e, timestamp = False, terminal=True)
 
 def show_details(o_inst):
     details = {
diff --git a/onionr/utils/detectoptimization.py b/onionr/utils/detectoptimization.py
new file mode 100755
index 00000000..20570242
--- /dev/null
+++ b/onionr/utils/detectoptimization.py
@@ -0,0 +1,27 @@
+'''
+    Onionr - Private P2P Communication
+
+    Detect if Python is being run in optimized mode or not, which has security considerations for assert statements
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+def detect_optimization():
+    '''Returns true if Python is run in optimized mode (-o), based on optimization ignoring assert statements'''
+    try:
+        assert True is False
+    except AssertionError:
+        return False
+    return True
\ No newline at end of file

From dce453db81a200eed06f04ca60d6a5068f48520f Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Thu, 27 Jun 2019 13:45:28 -0500
Subject: [PATCH 25/29] fixed output in onionr.py, work on docs

---
 docs/README.md                     | 16 ++++++++++++++++
 docs/{ => dev}/specs/block-spec.md |  0
 onionr/onionr.py                   |  4 ++--
 3 files changed, 18 insertions(+), 2 deletions(-)
 rename docs/{ => dev}/specs/block-spec.md (100%)

diff --git a/docs/README.md b/docs/README.md
index 343c6c17..6c5f44e5 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -1,3 +1,19 @@
 # Onionr Documentation
 
 The Onionr [whitepaper](whitepaper.md) is the best place to start both for users and developers.
+
+## User Documentation
+
+* [Installation](usage/install.md)
+* [First steps](usage/firststeps.md)
+* [Using Onionr Mail](usage/mail.md)
+* [Using Onionr web pages](usage/pages.md)
+* [Staying safe/anonymous](usage/safety.md)
+
+## Developer Documentation
+
+* [Development environment setup](dev/setup.md)
+* [Technical overview](dev/overview.md)
+* [Project layout](dev/layout.md)
+* [Plugin development guide](dev/plugins.md)
+* [Testing](dev/testing.md)
\ No newline at end of file
diff --git a/docs/specs/block-spec.md b/docs/dev/specs/block-spec.md
similarity index 100%
rename from docs/specs/block-spec.md
rename to docs/dev/specs/block-spec.md
diff --git a/onionr/onionr.py b/onionr/onionr.py
index befbf439..6ca5002e 100755
--- a/onionr/onionr.py
+++ b/onionr/onionr.py
@@ -236,13 +236,13 @@ class Onionr:
     def listPeers(self):
         logger.info('Peer transport address list:')
         for i in self.onionrCore.listAdders():
-            logger.info(i)
+            logger.info(i, terminal=True)
 
     def getWebPassword(self):
         return config.get('client.webpassword')
 
     def printWebPassword(self):
-        logger.info(self.getWebPassword(), term_only = True)
+        logger.info(self.getWebPassword(), terminal=True)
 
     def getHelp(self):
         return self.cmdhelp

From 03fbf32af91b51c5fc4b37d24a85d87f54e66f99 Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Thu, 27 Jun 2019 21:43:50 -0500
Subject: [PATCH 26/29] added reconstructhash for working with block hashes
 without their leading zeroes

---
 docs/usage/install.md               |  0
 onionr/static-data/www/mail/mail.js |  2 +-
 onionr/tests/test_zfill.py          | 21 ++++++++++++++
 onionr/utils/reconstructhash.py     | 43 +++++++++++++++++++++++++++++
 4 files changed, 65 insertions(+), 1 deletion(-)
 create mode 100644 docs/usage/install.md
 create mode 100644 onionr/tests/test_zfill.py
 create mode 100644 onionr/utils/reconstructhash.py

diff --git a/docs/usage/install.md b/docs/usage/install.md
new file mode 100644
index 00000000..e69de29b
diff --git a/onionr/static-data/www/mail/mail.js b/onionr/static-data/www/mail/mail.js
index 27180173..d3762135 100755
--- a/onionr/static-data/www/mail/mail.js
+++ b/onionr/static-data/www/mail/mail.js
@@ -77,7 +77,7 @@ function openThread(bHash, sender, date, sigBool, pubkey, subjectLine){
     var sigMsg = 'signature'
 
     // show add unknown contact button if peer is unknown but still has pubkey
-    if (sender === pubkey && sender !== myPub){
+    if (sender === pubkey && sender !== myPub && sigBool){
         addUnknownContact.style.display = 'inline'
     }
 
diff --git a/onionr/tests/test_zfill.py b/onionr/tests/test_zfill.py
new file mode 100644
index 00000000..e13ce1bb
--- /dev/null
+++ b/onionr/tests/test_zfill.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+import unittest, sys
+sys.path.append(".")
+
+from utils import reconstructhash
+
+class ZFill_Hash(unittest.TestCase):
+    def test_reconstruct(self):
+        h = b"4d20d791cbc293999b97cc627aa011692d317dede3d0fbd390c763210b0d"
+        self.assertEqual(reconstructhash.reconstruct_hash(h), b"0000" + h)
+        h = b"4d20d791cbc293999b97cc627aa011692d317dede3d0fbd390c763210b0d"
+        self.assertEqual(reconstructhash.reconstruct_hash(h, 62), b"00" + h)
+    
+    def test_deconstruct(self):
+        h = b"0000e918d24999ad9b0ff00c1d414f36b74afc93871a0ece4bd452f82b56af87"
+        h_no = b"e918d24999ad9b0ff00c1d414f36b74afc93871a0ece4bd452f82b56af87"
+        self.assertEqual(reconstructhash.deconstruct_hash(h), h_no)
+        h = "0000e918d24999ad9b0ff00c1d414f36b74afc93871a0ece4bd452f82b56af87"
+        h_no = "e918d24999ad9b0ff00c1d414f36b74afc93871a0ece4bd452f82b56af87"
+        self.assertEqual(reconstructhash.deconstruct_hash(h), h_no)
+unittest.main()
\ No newline at end of file
diff --git a/onionr/utils/reconstructhash.py b/onionr/utils/reconstructhash.py
new file mode 100644
index 00000000..ae50c0b2
--- /dev/null
+++ b/onionr/utils/reconstructhash.py
@@ -0,0 +1,43 @@
+'''
+    Onionr - Private P2P Communication
+
+    z-fill (zero fill) a string to a specific length, intended for reconstructing block hashes
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+def reconstruct_hash(hex_hash, length=64):
+    return hex_hash.zfill(length)
+
+def deconstruct_hash(hex_hash):
+    new_hash = ''
+    ret_bytes = False
+    try:
+        hex_hash = hex_hash.decode()
+        ret_bytes = True
+    except AttributeError:
+        pass
+
+    c = 0
+    for x in hex_hash:
+        if x == '0':
+            c += 1
+        else:
+            break
+    new_hash = hex_hash[c:]
+
+    if ret_bytes:
+
+        new_hash = new_hash.encode()
+    return new_hash
\ No newline at end of file

From a27a5e5a4551a2165e3c145f454f57a7e6143671 Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Fri, 28 Jun 2019 00:00:38 -0500
Subject: [PATCH 27/29] added support for 0-truncated hashes in
 stringvalidators, added test for hash validation

---
 onionr/onionrutils/stringvalidators.py |  4 +++-
 onionr/tests/test_stringvalidations.py | 19 +++++++++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/onionr/onionrutils/stringvalidators.py b/onionr/onionrutils/stringvalidators.py
index 5b7fbf6f..951c25ed 100644
--- a/onionr/onionrutils/stringvalidators.py
+++ b/onionr/onionrutils/stringvalidators.py
@@ -23,12 +23,14 @@ from onionrutils import bytesconverter
 def validate_hash(data, length=64):
     '''
         Validate if a string is a valid hash hex digest (does not compare, just checks length and charset)
+
+        Length is only invalid if its *more* than the specified
     '''
     retVal = True
     if data == False or data == True:
         return False
     data = data.strip()
-    if len(data) != length:
+    if len(data) > length:
         retVal = False
     else:
         try:
diff --git a/onionr/tests/test_stringvalidations.py b/onionr/tests/test_stringvalidations.py
index 72d5fb20..eef418ba 100755
--- a/onionr/tests/test_stringvalidations.py
+++ b/onionr/tests/test_stringvalidations.py
@@ -27,6 +27,25 @@ class OnionrValidations(unittest.TestCase):
             print('testing', x)
             self.assertFalse(stringvalidators.validate_transport(x))
     
+    def test_hash_validator(self):
+        valid = ['00003b3813a166e706e490238e9515633cc3d083efe982a67753d50d87a00c96\n', '00003b3813a166e706e490238e9515633cc3d083efe982a67753d50d87a00c96', b'00003b3813a166e706e490238e9515633cc3d083efe982a67753d50d87a00c96',
+        '00003b3813a166e706e490238e9515633cc36', b'00003b3813a166e706e490238e9515633cc3d083']
+        invalid = [None, 0, 1, True, False, '%#W483242#', '00003b3813a166e706e490238e9515633cc3d083efe982a67753d50d87a00c9666', '', b'', 
+        b'00003b3813a166e706e490238e9515633cc3d083efe982a67753d50d87a00c9666666', b'    ', '\n', '00003b3813a166e706e490238e9515633cc3d083efe982a67753d50d87a00ccccc\n']
+        
+        for x in valid:
+            self.assertTrue(stringvalidators.validate_hash(x))
+        for x in invalid:
+            try:
+                result = stringvalidators.validate_hash(x)
+                print('testing', x, result)
+            except AttributeError:
+                result = False
+            try:
+                self.assertFalse(result)
+            except AssertionError:
+                raise AssertionError("%s returned true" % (x,))
+    
     def test_pubkey_validator(self):
         # Test ed25519 public key validity
         valids = ['JZ5VE72GUS3C7BOHDRIYZX4B5U5EJMCMLKHLYCVBQQF3UKHYIRRQ====', 'JZ5VE72GUS3C7BOHDRIYZX4B5U5EJMCMLKHLYCVBQQF3UKHYIRRQ']

From bc8080765d45128390c265f84ba3991430bd0f51 Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Fri, 28 Jun 2019 15:23:03 -0500
Subject: [PATCH 28/29] work on docs, added firststeps.md

---
 docs/{ => dev}/http-api.md |  0
 docs/usage/firststeps.md   |  7 +++++++
 docs/usage/install.md      | 13 +++++++++++++
 3 files changed, 20 insertions(+)
 rename docs/{ => dev}/http-api.md (100%)
 create mode 100644 docs/usage/firststeps.md

diff --git a/docs/http-api.md b/docs/dev/http-api.md
similarity index 100%
rename from docs/http-api.md
rename to docs/dev/http-api.md
diff --git a/docs/usage/firststeps.md b/docs/usage/firststeps.md
new file mode 100644
index 00000000..8733565d
--- /dev/null
+++ b/docs/usage/firststeps.md
@@ -0,0 +1,7 @@
+# Onionr First Steps
+
+After installing Onionr, there are several things to do:
+
+1. Setup a [deterministic address](usage/deterministic.md) (optional)
+2. Add friends' ids
+3. Publish your id
\ No newline at end of file
diff --git a/docs/usage/install.md b/docs/usage/install.md
index e69de29b..f1fb6130 100644
--- a/docs/usage/install.md
+++ b/docs/usage/install.md
@@ -0,0 +1,13 @@
+# Onionr Installation
+
+The following steps work broadly speaking for Windows, Mac, and Linux.
+
+1. Verify python3.6+ is installed: if its not see https://www.python.org/downloads/
+
+2. Verify Tor is installed (does not need to be running, binary can be put into system path or Onionr directory)
+
+3. [Optional but recommended]: setup virtual environment using [virtualenv](https://virtualenv.pypa.io/en/latest/), activate the virtual environment
+
+4. Clone Onionr: git clone https://gitlab.com/beardog/onionr
+
+5. Install the Python module dependencies: pip3 install --require-hashes -r requirements.txt

From bb5378ec4caed494f8afd1f3a13eac4cc2212586 Mon Sep 17 00:00:00 2001
From: Kevin Froman <beardog@firemail.cc>
Date: Fri, 28 Jun 2019 16:10:29 -0500
Subject: [PATCH 29/29] fix importblocks metadata call not having core inst

---
 onionr/onionrutils/importnewblocks.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/onionr/onionrutils/importnewblocks.py b/onionr/onionrutils/importnewblocks.py
index 6accd947..e1793adf 100644
--- a/onionr/onionrutils/importnewblocks.py
+++ b/onionr/onionrutils/importnewblocks.py
@@ -35,14 +35,14 @@ def import_new_blocks(core_inst=None, scanDir=''):
     for block in glob.glob(scanDir + "*.dat"):
         if block.replace(scanDir, '').replace('.dat', '') not in blockList:
             exist = True
-            logger.info('Found new block on dist %s' % block)
+            logger.info('Found new block on dist %s' % block, terminal=True)
             with open(block, 'rb') as newBlock:
                 block = block.replace(scanDir, '').replace('.dat', '')
                 if core_inst._crypto.sha3Hash(newBlock.read()) == block.replace('.dat', ''):
                     core_inst.addToBlockDB(block.replace('.dat', ''), dataSaved=True)
-                    logger.info('Imported block %s.' % block)
-                    blockmetadata.process_block_metadata(block)
+                    logger.info('Imported block %s.' % block, terminal=True)
+                    blockmetadata.process_block_metadata(core_inst, block)
                 else:
-                    logger.warn('Failed to verify hash for %s' % block)
+                    logger.warn('Failed to verify hash for %s' % block, terminal=True)
     if not exist:
-        logger.info('No blocks found to import')
\ No newline at end of file
+        logger.info('No blocks found to import', terminal=True)
\ No newline at end of file