improved block list syncing, added forgotten stats.js file

onionr/api.py

@@ -109,7 +109,8 @@ class PublicAPI:
 
         @app.route('/getblocklist')
         def getBlockList():
-            bList = clientAPI._core.getBlockList()
+            dateAdjust = request.args.get('date')
+            bList = clientAPI._core.getBlockList(dateRec=dateAdjust)
             for b in self.hideBlocks:
                 if b in bList:
                     bList.remove(b)
@@ -304,7 +305,6 @@ class API:
 
         @app.route('/queueResponseAdd/<name>', methods=['post'])
         def queueResponseAdd(name):
-            print('added',name)
             self.queueResponse[name] = request.form['data']
             return Response('success')
         
@@ -317,7 +317,6 @@ class API:
                 pass
             else:
                 del self.queueResponse[name]
-            print(name, resp)
             return Response(resp)
             
         @app.route('/ping')
@@ -381,6 +380,12 @@ class API:
                 pass
             return Response("bye")
 
+        @app.route('/shutdownclean')
+        def shutdownClean():
+            # good for calling from other clients
+            self._core.daemonQueueAdd('shutdown')
+            return Response("bye")
+        
         @app.route('/getstats')
         def getStats():
             return Response(self._core.serializer.getStats())

onionr/communicator2.py

@@ -21,7 +21,7 @@
 '''
 import sys, os, core, config, json, requests, time, logger, threading, base64, onionr, uuid
 import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block
-import onionrdaemontools, onionrsockets, onionrchat, onionr, onionrproofs
+import onionrdaemontools, onionrsockets, onionr, onionrproofs, proofofmemory
 import binascii
 from dependencies import secrets
 from defusedxml import minidom
@@ -74,6 +74,9 @@ class OnionrCommunicatorDaemon:
         # timestamp when the last online node was seen
         self.lastNodeSeen = None
 
+        # Dict of time stamps for peer's block list lookup times, to avoid downloading full lists all the time
+        self.dbTimestamps = {}
+
         # Clear the daemon queue for any dead messages
         if os.path.exists(self._core.queueDB):
             self._core.clearDaemonQueue()
@@ -81,11 +84,12 @@ class OnionrCommunicatorDaemon:
         # Loads in and starts the enabled plugins
         plugins.reload()
 
+        self.proofofmemory = proofofmemory.ProofOfMemory(self)
+
         # daemon tools are misc daemon functions, e.g. announce to online peers
         # intended only for use by OnionrCommunicatorDaemon
         self.daemonTools = onionrdaemontools.DaemonTools(self)
 
-        self._chat = onionrchat.OnionrChat(self)
 
         if debug or developmentMode:
             OnionrCommunicatorTimers(self, self.heartbeat, 30)
@@ -164,7 +168,9 @@ class OnionrCommunicatorDaemon:
         existingBlocks = self._core.getBlockList()
         triedPeers = [] # list of peers we've tried this time around
         maxBacklog = 1560 # Max amount of *new* block hashes to have already in queue, to avoid memory exhaustion
+        lastLookupTime = 0 # Last time we looked up a particular peer's list
         for i in range(tryAmount):
+            listLookupCommand = 'getblocklist' # This is defined here to reset it each time
             if len(self.blockQueue) >= maxBacklog:
                 break
             if not self.isOnline:
@@ -186,11 +192,21 @@ class OnionrCommunicatorDaemon:
             triedPeers.append(peer)
             if newDBHash != self._core.getAddressInfo(peer, 'DBHash'):
                 self._core.setAddressInfo(peer, 'DBHash', newDBHash)
+                # Get the last time we looked up a peer's stamp to only fetch blocks since then.
+                # Saved in memory only for privacy reasons
+                try:
+                    lastLookupTime = self.dbTimestamps[peer]
+                except KeyError:
+                    lastLookupTime = 0
+                else:
+                    listLookupCommand += '?date=%s' % (lastLookupTime,)
                 try:
                     newBlocks = self.peerAction(peer, 'getblocklist') # get list of new block hashes
                 except Exception as error:
                     logger.warn('Could not get new blocks from %s.' % peer, error = error)
                     newBlocks = False
+                else:
+                    self.dbTimestamps[peer] = self._core._utils.getRoundedEpoch(roundS=60)
                 if newBlocks != False:
                     # if request was a success
                     for i in newBlocks.split('\n'):
@@ -224,8 +240,8 @@ class OnionrCommunicatorDaemon:
             if self._core._utils.storageCounter.isFull():
                 break
             self.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block
-            logger.info("Attempting to download %s..." % blockHash)
             peerUsed = self.pickOnlinePeer()
+            logger.info("Attempting to download %s from %s..." % (blockHash[:12], peerUsed))
             content = self.peerAction(peerUsed, 'getdata/' + blockHash) # block content from random peer (includes metadata)
             if content != False and len(content) > 0:
                 try:
@@ -247,7 +263,7 @@ class OnionrCommunicatorDaemon:
                     metadata = metas[0]
                     if self._core._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid, and verify nonce
                         if self._core._crypto.verifyPow(content): # check if POW is enough/correct
-                            logger.info('Attempting to save block %s...' % blockHash)
+                            logger.info('Attempting to save block %s...' % blockHash[:12])
                             try:
                                 self._core.setData(content)
                             except onionrexceptions.DiskAllocationReached:
@@ -403,6 +419,10 @@ class OnionrCommunicatorDaemon:
             del self.connectTimes[peer]
         except KeyError:
             pass
+        try:
+            del self.dbTimestamps[peer]
+        except KeyError:
+            pass
         try:
             self.onlinePeers.remove(peer)
         except ValueError:

onionr/core.py

@@ -21,7 +21,7 @@ import sqlite3, os, sys, time, math, base64, tarfile, nacl, logger, json, netcon
 from onionrblockapi import Block
 
 import onionrutils, onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions, onionrvalues
-import onionrblacklist, onionrchat, onionrusers
+import onionrblacklist, onionrusers
 import dbcreator, onionrstorage, serializeddata
 
 if sys.version_info < (3, 6):
@@ -373,11 +373,11 @@ class Core:
         try:
             c.execute('INSERT INTO commands (command, data, date, responseID) VALUES(?, ?, ?, ?)', t)
             conn.commit()
-            conn.close()
         except sqlite3.OperationalError:
             retData = False
             self.daemonQueue()
         events.event('queue_push', data = {'command': command, 'data': data}, onionr = None)
+        conn.close()
         return retData
     
     def daemonQueueGetResponse(self, responseID=''):
@@ -602,24 +602,26 @@ class Core:
 
         return
 
-    def getBlockList(self, unsaved = False): # TODO: Use unsaved??
+    def getBlockList(self, dateRec = None, unsaved = False):
         '''
             Get list of our blocks
         '''
+        if dateRec == None:
+            dateRec = 0
 
         conn = sqlite3.connect(self.blockDB, timeout=10)
         c = conn.cursor()
 
-        if unsaved:
+        # if unsaved:
-            execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();'
+        #     execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();'
-        else:
+        # else:
-            execute = 'SELECT hash FROM hashes ORDER BY dateReceived ASC;'
+        #     execute = 'SELECT hash FROM hashes ORDER BY dateReceived ASC;'
-
+        execute = 'SELECT hash FROM hashes WHERE dateReceived >= ? ORDER BY dateReceived ASC;'
+        args = (dateRec,)
         rows = list()
-        for row in c.execute(execute):
+        for row in c.execute(execute, args):
             for i in row:
                 rows.append(i)
-
         return rows
 
     def getBlockDate(self, blockHash):

onionr/onionr.py

@@ -21,8 +21,9 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 import sys
-if sys.version_info[0] == 2 or sys.version_info[1] < 5:
+MIN_PY_VERSION = 6
-    print('Error, Onionr requires Python 3.5+')
+if sys.version_info[0] == 2 or sys.version_info[1] < MIN_PY_VERSION:
+    print('Error, Onionr requires Python 3.%s+' % (MIN_PY_VERSION,))
     sys.exit(1)
 import os, base64, random, getpass, shutil, subprocess, requests, time, platform, datetime, re, json, getpass, sqlite3
 import webbrowser, uuid, signal
@@ -198,7 +199,6 @@ class Onionr:
 
             'ui' : self.openUI,
             'gui' : self.openUI,
-            'chat': self.startChat,
 
             'getpassword': self.printWebPassword,
             'get-password': self.printWebPassword,
@@ -209,8 +209,6 @@ class Onionr:
             'getpasswd': self.printWebPassword,
             'get-passwd': self.printWebPassword,
 
-            'chat': self.startChat,
-
             'friend': self.friendCmd,
             'add-id': self.addID,
             'change-id': self.changeID
@@ -328,14 +326,6 @@ class Onionr:
             else:
                 logger.error('Invalid key %s' % (key,))
 
-    def startChat(self):
-        try:
-            data = json.dumps({'peer': sys.argv[2], 'reason': 'chat'})
-        except IndexError:
-            logger.error('Must specify peer to chat with.')
-        else:
-            self.onionrCore.daemonQueueAdd('startSocket', data)
-
     def getCommands(self):
         return self.cmds
 
@@ -811,7 +801,7 @@ class Onionr:
 
         try:
             # define stats messages here
-            totalBlocks = len(Block.getBlocks())
+            totalBlocks = len(self.onionrCore.getBlockList())
             signedBlocks = len(Block.getBlocks(signed = True))
             messages = {
                 # info about local client

onionr/onionrchat.py

@@ -1,50 +0,0 @@
-'''
-    Onionr - P2P Anonymous Storage Network
-
-    Onionr Chat Messages
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import logger, time
-
-class OnionrChat:
-    def __init__(self, communicatorInst):
-        '''OnionrChat uses onionrsockets (handled by the communicator) to exchange direct chat messages'''
-        self.communicator = communicatorInst
-        self._core = self.communicator._core
-        self._utils = self._core._utils
-
-        self.chats = {} # {'peer': {'date': date, message': message}}
-        self.chatSend = {}
-
-    def chatHandler(self):
-        while not self.communicator.shutdown:
-            for peer in self._core.socketServerConnData:
-                try:
-                    assert self._core.socketReasons[peer] == "chat"
-                except (AssertionError, KeyError) as e:
-                    logger.warn('Peer is not for chat')
-                    continue
-                else:
-                    self.chats[peer] = {'date': self._core.socketServerConnData[peer]['date'], 'data': self._core.socketServerConnData[peer]['data']}
-                    logger.info("CHAT MESSAGE RECIEVED: %s" % self.chats[peer]['data'])
-            for peer in self.communicator.socketClient.sockets:
-                try:
-                    logger.info(self.communicator.socketClient.connPool[peer]['data'])
-                    self.communicator.socketClient.sendData(peer, "lol")
-                except:
-                    pass
-
-            time.sleep(2)

onionr/onionrutils.py

@@ -183,7 +183,6 @@ class OnionrUtils:
         if data != '':
             data = '&data=' + urllib.parse.quote_plus(data)
         payload = 'http://%s/%s%s' % (hostname, command, data)
-        print(payload,config.get('client.webpassword'))
         try:
             if post:
                 retData = requests.post(payload, data=postData, headers={'token': config.get('client.webpassword')}, timeout=(15, 30)).text

onionr/static-data/default_config.json

@@ -2,8 +2,8 @@
     "general" : {
         "dev_mode" : true,
         "display_header" : false,
-        "minimum_block_pow": 3,
+        "minimum_block_pow": 1,
-        "minimum_send_pow": 3,
+        "minimum_send_pow": 1,
         "socket_servers": false,
         "security_level": 0,
         "max_block_age": 2678400,

onionr/static-data/www/private/index.html

@@ -5,20 +5,27 @@
         <title>
             Onionr
         </title>
+        <link rel='stylesheet' href='/shared/style/modal.css'>
         <link rel='stylesheet' href='/shared/main/style.css'>
     </head>
     <body>
+        <div id="shutdownNotice" class='overlay'>
+            <div>
+                <p>Your node will shutdown. Thank you for using Onionr.</p>
+            </div>
+        </div>
         <img class='logo' src='/shared/onionr-icon.png' alt='onionr logo'>
         <span class='logoText'>Onionr Web Control Panel</span>
         <div class='content'>
-            <a href='shutdown'>Shutdown node</a>
+            <button id='shutdownNode'>Shutdown Node</button>
             <h2>Stats</h2>
             <p>Uptime: <span id='uptime'></span></p>
             <p>Stored Blocks: <span id='storedBlocks'></span></p>
             <p>Connected nodes:</p>
             <pre id='connectedNodes'></pre>
         </div>
+        <script src='/shared/misc.js'></script>
+        <script src='/shared/main/stats.js'></script>
+        <script src='/shared/panel.js'></script>
     </body>
-    <script src='/shared/misc.js'></script>
-    <script src='/shared/main/stats.js'></script>
 </html>

onionr/static-data/www/shared/main/stats.js

@@ -17,14 +17,11 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>
 */
-
 uptimeDisplay = document.getElementById('uptime')
 connectedDisplay = document.getElementById('connectedNodes')
 storedBlockDisplay = document.getElementById('storedBlocks')
 
-pass = window.location.hash.replace('#', '')
+stats = JSON.parse(httpGet('getstats', webpass))
-
-stats = JSON.parse(httpGet('getstats', pass))
 uptimeDisplay.innerText = stats['uptime'] + ' seconds'
 connectedDisplay.innerText = stats['connectedNodes']
 storedBlockDisplay.innerText = stats['blockCount']

onionr/static-data/www/shared/main/style.css

@@ -124,3 +124,17 @@ body{
         display: block;
     }
 }
+
+/*https://stackoverflow.com/a/16778646*/
+.overlay {
+    visibility: hidden;
+    position: absolute;
+    left: 0px;
+    top: 0px;
+    width:100%;
+    opacity: 0.9;
+    height:100%;
+    text-align:center;
+    z-index: 1000;
+    background-color: black;
+ }

onionr/static-data/www/shared/misc.js

@@ -1,4 +1,16 @@
-function httpGet(theUrl, webpass) {
+webpass = document.location.hash.replace('#', '')
+if (typeof webpass == "undefined"){
+    webpass = localStorage['webpass']
+}
+else{
+    localStorage['webpass'] = webpass
+    document.location.hash = ''
+}
+if (typeof webpass == "undefined" || webpass == ""){
+    alert('Web password was not found in memory or URL')
+}
+
+function httpGet(theUrl) {
     var xmlHttp = new XMLHttpRequest()
     xmlHttp.open( "GET", theUrl, false ) // false for synchronous request
     xmlHttp.setRequestHeader('token', webpass)
@@ -7,6 +19,10 @@ function httpGet(theUrl, webpass) {
         return xmlHttp.responseText
     }
     else{
-        return "";
+        return ""
     }
 }
+function overlay(overlayID) {
+    el = document.getElementById(overlayID)
+   el.style.visibility = (el.style.visibility == "visible") ? "hidden" : "visible"
+ }

onionr/static-data/www/shared/panel.js

@@ -0,0 +1,6 @@
+shutdownBtn = document.getElementById('shutdownNode')
+
+shutdownBtn.onclick = function(){
+    httpGet('shutdownclean')
+    overlay('shutdownNotice')
+}