improved block list syncing, added forgotten stats.js file

This commit is contained in:
Kevin Froman 2019-01-15 23:57:47 -06:00
parent 0e6ab04996
commit 1ebed8d606
12 changed files with 100 additions and 94 deletions

View File

@ -109,7 +109,8 @@ class PublicAPI:
@app.route('/getblocklist') @app.route('/getblocklist')
def getBlockList(): def getBlockList():
bList = clientAPI._core.getBlockList() dateAdjust = request.args.get('date')
bList = clientAPI._core.getBlockList(dateRec=dateAdjust)
for b in self.hideBlocks: for b in self.hideBlocks:
if b in bList: if b in bList:
bList.remove(b) bList.remove(b)
@ -304,7 +305,6 @@ class API:
@app.route('/queueResponseAdd/<name>', methods=['post']) @app.route('/queueResponseAdd/<name>', methods=['post'])
def queueResponseAdd(name): def queueResponseAdd(name):
print('added',name)
self.queueResponse[name] = request.form['data'] self.queueResponse[name] = request.form['data']
return Response('success') return Response('success')
@ -317,7 +317,6 @@ class API:
pass pass
else: else:
del self.queueResponse[name] del self.queueResponse[name]
print(name, resp)
return Response(resp) return Response(resp)
@app.route('/ping') @app.route('/ping')
@ -381,6 +380,12 @@ class API:
pass pass
return Response("bye") return Response("bye")
@app.route('/shutdownclean')
def shutdownClean():
# good for calling from other clients
self._core.daemonQueueAdd('shutdown')
return Response("bye")
@app.route('/getstats') @app.route('/getstats')
def getStats(): def getStats():
return Response(self._core.serializer.getStats()) return Response(self._core.serializer.getStats())

View File

@ -21,7 +21,7 @@
''' '''
import sys, os, core, config, json, requests, time, logger, threading, base64, onionr, uuid import sys, os, core, config, json, requests, time, logger, threading, base64, onionr, uuid
import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block
import onionrdaemontools, onionrsockets, onionrchat, onionr, onionrproofs import onionrdaemontools, onionrsockets, onionr, onionrproofs, proofofmemory
import binascii import binascii
from dependencies import secrets from dependencies import secrets
from defusedxml import minidom from defusedxml import minidom
@ -74,6 +74,9 @@ class OnionrCommunicatorDaemon:
# timestamp when the last online node was seen # timestamp when the last online node was seen
self.lastNodeSeen = None self.lastNodeSeen = None
# Dict of time stamps for peer's block list lookup times, to avoid downloading full lists all the time
self.dbTimestamps = {}
# Clear the daemon queue for any dead messages # Clear the daemon queue for any dead messages
if os.path.exists(self._core.queueDB): if os.path.exists(self._core.queueDB):
self._core.clearDaemonQueue() self._core.clearDaemonQueue()
@ -81,11 +84,12 @@ class OnionrCommunicatorDaemon:
# Loads in and starts the enabled plugins # Loads in and starts the enabled plugins
plugins.reload() plugins.reload()
self.proofofmemory = proofofmemory.ProofOfMemory(self)
# daemon tools are misc daemon functions, e.g. announce to online peers # daemon tools are misc daemon functions, e.g. announce to online peers
# intended only for use by OnionrCommunicatorDaemon # intended only for use by OnionrCommunicatorDaemon
self.daemonTools = onionrdaemontools.DaemonTools(self) self.daemonTools = onionrdaemontools.DaemonTools(self)
self._chat = onionrchat.OnionrChat(self)
if debug or developmentMode: if debug or developmentMode:
OnionrCommunicatorTimers(self, self.heartbeat, 30) OnionrCommunicatorTimers(self, self.heartbeat, 30)
@ -164,7 +168,9 @@ class OnionrCommunicatorDaemon:
existingBlocks = self._core.getBlockList() existingBlocks = self._core.getBlockList()
triedPeers = [] # list of peers we've tried this time around triedPeers = [] # list of peers we've tried this time around
maxBacklog = 1560 # Max amount of *new* block hashes to have already in queue, to avoid memory exhaustion maxBacklog = 1560 # Max amount of *new* block hashes to have already in queue, to avoid memory exhaustion
lastLookupTime = 0 # Last time we looked up a particular peer's list
for i in range(tryAmount): for i in range(tryAmount):
listLookupCommand = 'getblocklist' # This is defined here to reset it each time
if len(self.blockQueue) >= maxBacklog: if len(self.blockQueue) >= maxBacklog:
break break
if not self.isOnline: if not self.isOnline:
@ -186,11 +192,21 @@ class OnionrCommunicatorDaemon:
triedPeers.append(peer) triedPeers.append(peer)
if newDBHash != self._core.getAddressInfo(peer, 'DBHash'): if newDBHash != self._core.getAddressInfo(peer, 'DBHash'):
self._core.setAddressInfo(peer, 'DBHash', newDBHash) self._core.setAddressInfo(peer, 'DBHash', newDBHash)
# Get the last time we looked up a peer's stamp to only fetch blocks since then.
# Saved in memory only for privacy reasons
try:
lastLookupTime = self.dbTimestamps[peer]
except KeyError:
lastLookupTime = 0
else:
listLookupCommand += '?date=%s' % (lastLookupTime,)
try: try:
newBlocks = self.peerAction(peer, 'getblocklist') # get list of new block hashes newBlocks = self.peerAction(peer, 'getblocklist') # get list of new block hashes
except Exception as error: except Exception as error:
logger.warn('Could not get new blocks from %s.' % peer, error = error) logger.warn('Could not get new blocks from %s.' % peer, error = error)
newBlocks = False newBlocks = False
else:
self.dbTimestamps[peer] = self._core._utils.getRoundedEpoch(roundS=60)
if newBlocks != False: if newBlocks != False:
# if request was a success # if request was a success
for i in newBlocks.split('\n'): for i in newBlocks.split('\n'):
@ -224,8 +240,8 @@ class OnionrCommunicatorDaemon:
if self._core._utils.storageCounter.isFull(): if self._core._utils.storageCounter.isFull():
break break
self.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block self.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block
logger.info("Attempting to download %s..." % blockHash)
peerUsed = self.pickOnlinePeer() peerUsed = self.pickOnlinePeer()
logger.info("Attempting to download %s from %s..." % (blockHash[:12], peerUsed))
content = self.peerAction(peerUsed, 'getdata/' + blockHash) # block content from random peer (includes metadata) content = self.peerAction(peerUsed, 'getdata/' + blockHash) # block content from random peer (includes metadata)
if content != False and len(content) > 0: if content != False and len(content) > 0:
try: try:
@ -247,7 +263,7 @@ class OnionrCommunicatorDaemon:
metadata = metas[0] metadata = metas[0]
if self._core._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid, and verify nonce if self._core._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid, and verify nonce
if self._core._crypto.verifyPow(content): # check if POW is enough/correct if self._core._crypto.verifyPow(content): # check if POW is enough/correct
logger.info('Attempting to save block %s...' % blockHash) logger.info('Attempting to save block %s...' % blockHash[:12])
try: try:
self._core.setData(content) self._core.setData(content)
except onionrexceptions.DiskAllocationReached: except onionrexceptions.DiskAllocationReached:
@ -403,6 +419,10 @@ class OnionrCommunicatorDaemon:
del self.connectTimes[peer] del self.connectTimes[peer]
except KeyError: except KeyError:
pass pass
try:
del self.dbTimestamps[peer]
except KeyError:
pass
try: try:
self.onlinePeers.remove(peer) self.onlinePeers.remove(peer)
except ValueError: except ValueError:

View File

@ -21,7 +21,7 @@ import sqlite3, os, sys, time, math, base64, tarfile, nacl, logger, json, netcon
from onionrblockapi import Block from onionrblockapi import Block
import onionrutils, onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions, onionrvalues import onionrutils, onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions, onionrvalues
import onionrblacklist, onionrchat, onionrusers import onionrblacklist, onionrusers
import dbcreator, onionrstorage, serializeddata import dbcreator, onionrstorage, serializeddata
if sys.version_info < (3, 6): if sys.version_info < (3, 6):
@ -373,11 +373,11 @@ class Core:
try: try:
c.execute('INSERT INTO commands (command, data, date, responseID) VALUES(?, ?, ?, ?)', t) c.execute('INSERT INTO commands (command, data, date, responseID) VALUES(?, ?, ?, ?)', t)
conn.commit() conn.commit()
conn.close()
except sqlite3.OperationalError: except sqlite3.OperationalError:
retData = False retData = False
self.daemonQueue() self.daemonQueue()
events.event('queue_push', data = {'command': command, 'data': data}, onionr = None) events.event('queue_push', data = {'command': command, 'data': data}, onionr = None)
conn.close()
return retData return retData
def daemonQueueGetResponse(self, responseID=''): def daemonQueueGetResponse(self, responseID=''):
@ -602,24 +602,26 @@ class Core:
return return
def getBlockList(self, unsaved = False): # TODO: Use unsaved?? def getBlockList(self, dateRec = None, unsaved = False):
''' '''
Get list of our blocks Get list of our blocks
''' '''
if dateRec == None:
dateRec = 0
conn = sqlite3.connect(self.blockDB, timeout=10) conn = sqlite3.connect(self.blockDB, timeout=10)
c = conn.cursor() c = conn.cursor()
if unsaved: # if unsaved:
execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();' # execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();'
else: # else:
execute = 'SELECT hash FROM hashes ORDER BY dateReceived ASC;' # execute = 'SELECT hash FROM hashes ORDER BY dateReceived ASC;'
execute = 'SELECT hash FROM hashes WHERE dateReceived >= ? ORDER BY dateReceived ASC;'
args = (dateRec,)
rows = list() rows = list()
for row in c.execute(execute): for row in c.execute(execute, args):
for i in row: for i in row:
rows.append(i) rows.append(i)
return rows return rows
def getBlockDate(self, blockHash): def getBlockDate(self, blockHash):

View File

@ -21,8 +21,9 @@
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
import sys import sys
if sys.version_info[0] == 2 or sys.version_info[1] < 5: MIN_PY_VERSION = 6
print('Error, Onionr requires Python 3.5+') if sys.version_info[0] == 2 or sys.version_info[1] < MIN_PY_VERSION:
print('Error, Onionr requires Python 3.%s+' % (MIN_PY_VERSION,))
sys.exit(1) sys.exit(1)
import os, base64, random, getpass, shutil, subprocess, requests, time, platform, datetime, re, json, getpass, sqlite3 import os, base64, random, getpass, shutil, subprocess, requests, time, platform, datetime, re, json, getpass, sqlite3
import webbrowser, uuid, signal import webbrowser, uuid, signal
@ -198,7 +199,6 @@ class Onionr:
'ui' : self.openUI, 'ui' : self.openUI,
'gui' : self.openUI, 'gui' : self.openUI,
'chat': self.startChat,
'getpassword': self.printWebPassword, 'getpassword': self.printWebPassword,
'get-password': self.printWebPassword, 'get-password': self.printWebPassword,
@ -209,8 +209,6 @@ class Onionr:
'getpasswd': self.printWebPassword, 'getpasswd': self.printWebPassword,
'get-passwd': self.printWebPassword, 'get-passwd': self.printWebPassword,
'chat': self.startChat,
'friend': self.friendCmd, 'friend': self.friendCmd,
'add-id': self.addID, 'add-id': self.addID,
'change-id': self.changeID 'change-id': self.changeID
@ -328,14 +326,6 @@ class Onionr:
else: else:
logger.error('Invalid key %s' % (key,)) logger.error('Invalid key %s' % (key,))
def startChat(self):
try:
data = json.dumps({'peer': sys.argv[2], 'reason': 'chat'})
except IndexError:
logger.error('Must specify peer to chat with.')
else:
self.onionrCore.daemonQueueAdd('startSocket', data)
def getCommands(self): def getCommands(self):
return self.cmds return self.cmds
@ -811,7 +801,7 @@ class Onionr:
try: try:
# define stats messages here # define stats messages here
totalBlocks = len(Block.getBlocks()) totalBlocks = len(self.onionrCore.getBlockList())
signedBlocks = len(Block.getBlocks(signed = True)) signedBlocks = len(Block.getBlocks(signed = True))
messages = { messages = {
# info about local client # info about local client

View File

@ -1,50 +0,0 @@
'''
Onionr - P2P Anonymous Storage Network
Onionr Chat Messages
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import logger, time
class OnionrChat:
def __init__(self, communicatorInst):
'''OnionrChat uses onionrsockets (handled by the communicator) to exchange direct chat messages'''
self.communicator = communicatorInst
self._core = self.communicator._core
self._utils = self._core._utils
self.chats = {} # {'peer': {'date': date, message': message}}
self.chatSend = {}
def chatHandler(self):
while not self.communicator.shutdown:
for peer in self._core.socketServerConnData:
try:
assert self._core.socketReasons[peer] == "chat"
except (AssertionError, KeyError) as e:
logger.warn('Peer is not for chat')
continue
else:
self.chats[peer] = {'date': self._core.socketServerConnData[peer]['date'], 'data': self._core.socketServerConnData[peer]['data']}
logger.info("CHAT MESSAGE RECIEVED: %s" % self.chats[peer]['data'])
for peer in self.communicator.socketClient.sockets:
try:
logger.info(self.communicator.socketClient.connPool[peer]['data'])
self.communicator.socketClient.sendData(peer, "lol")
except:
pass
time.sleep(2)

View File

@ -183,7 +183,6 @@ class OnionrUtils:
if data != '': if data != '':
data = '&data=' + urllib.parse.quote_plus(data) data = '&data=' + urllib.parse.quote_plus(data)
payload = 'http://%s/%s%s' % (hostname, command, data) payload = 'http://%s/%s%s' % (hostname, command, data)
print(payload,config.get('client.webpassword'))
try: try:
if post: if post:
retData = requests.post(payload, data=postData, headers={'token': config.get('client.webpassword')}, timeout=(15, 30)).text retData = requests.post(payload, data=postData, headers={'token': config.get('client.webpassword')}, timeout=(15, 30)).text

View File

@ -2,8 +2,8 @@
"general" : { "general" : {
"dev_mode" : true, "dev_mode" : true,
"display_header" : false, "display_header" : false,
"minimum_block_pow": 3, "minimum_block_pow": 1,
"minimum_send_pow": 3, "minimum_send_pow": 1,
"socket_servers": false, "socket_servers": false,
"security_level": 0, "security_level": 0,
"max_block_age": 2678400, "max_block_age": 2678400,

View File

@ -5,20 +5,27 @@
<title> <title>
Onionr Onionr
</title> </title>
<link rel='stylesheet' href='/shared/style/modal.css'>
<link rel='stylesheet' href='/shared/main/style.css'> <link rel='stylesheet' href='/shared/main/style.css'>
</head> </head>
<body> <body>
<div id="shutdownNotice" class='overlay'>
<div>
<p>Your node will shutdown. Thank you for using Onionr.</p>
</div>
</div>
<img class='logo' src='/shared/onionr-icon.png' alt='onionr logo'> <img class='logo' src='/shared/onionr-icon.png' alt='onionr logo'>
<span class='logoText'>Onionr Web Control Panel</span> <span class='logoText'>Onionr Web Control Panel</span>
<div class='content'> <div class='content'>
<a href='shutdown'>Shutdown node</a> <button id='shutdownNode'>Shutdown Node</button>
<h2>Stats</h2> <h2>Stats</h2>
<p>Uptime: <span id='uptime'></span></p> <p>Uptime: <span id='uptime'></span></p>
<p>Stored Blocks: <span id='storedBlocks'></span></p> <p>Stored Blocks: <span id='storedBlocks'></span></p>
<p>Connected nodes:</p> <p>Connected nodes:</p>
<pre id='connectedNodes'></pre> <pre id='connectedNodes'></pre>
</div> </div>
</body>
<script src='/shared/misc.js'></script> <script src='/shared/misc.js'></script>
<script src='/shared/main/stats.js'></script> <script src='/shared/main/stats.js'></script>
<script src='/shared/panel.js'></script>
</body>
</html> </html>

View File

@ -17,14 +17,11 @@
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/> along with this program. If not, see <https://www.gnu.org/licenses/>
*/ */
uptimeDisplay = document.getElementById('uptime') uptimeDisplay = document.getElementById('uptime')
connectedDisplay = document.getElementById('connectedNodes') connectedDisplay = document.getElementById('connectedNodes')
storedBlockDisplay = document.getElementById('storedBlocks') storedBlockDisplay = document.getElementById('storedBlocks')
pass = window.location.hash.replace('#', '') stats = JSON.parse(httpGet('getstats', webpass))
stats = JSON.parse(httpGet('getstats', pass))
uptimeDisplay.innerText = stats['uptime'] + ' seconds' uptimeDisplay.innerText = stats['uptime'] + ' seconds'
connectedDisplay.innerText = stats['connectedNodes'] connectedDisplay.innerText = stats['connectedNodes']
storedBlockDisplay.innerText = stats['blockCount'] storedBlockDisplay.innerText = stats['blockCount']

View File

@ -124,3 +124,17 @@ body{
display: block; display: block;
} }
} }
/*https://stackoverflow.com/a/16778646*/
.overlay {
visibility: hidden;
position: absolute;
left: 0px;
top: 0px;
width:100%;
opacity: 0.9;
height:100%;
text-align:center;
z-index: 1000;
background-color: black;
}

View File

@ -1,4 +1,16 @@
function httpGet(theUrl, webpass) { webpass = document.location.hash.replace('#', '')
if (typeof webpass == "undefined"){
webpass = localStorage['webpass']
}
else{
localStorage['webpass'] = webpass
document.location.hash = ''
}
if (typeof webpass == "undefined" || webpass == ""){
alert('Web password was not found in memory or URL')
}
function httpGet(theUrl) {
var xmlHttp = new XMLHttpRequest() var xmlHttp = new XMLHttpRequest()
xmlHttp.open( "GET", theUrl, false ) // false for synchronous request xmlHttp.open( "GET", theUrl, false ) // false for synchronous request
xmlHttp.setRequestHeader('token', webpass) xmlHttp.setRequestHeader('token', webpass)
@ -7,6 +19,10 @@ function httpGet(theUrl, webpass) {
return xmlHttp.responseText return xmlHttp.responseText
} }
else{ else{
return ""; return ""
} }
} }
function overlay(overlayID) {
el = document.getElementById(overlayID)
el.style.visibility = (el.style.visibility == "visible") ? "hidden" : "visible"
}

View File

@ -0,0 +1,6 @@
shutdownBtn = document.getElementById('shutdownNode')
shutdownBtn.onclick = function(){
httpGet('shutdownclean')
overlay('shutdownNotice')
}