kev
/
Onionr
1
0
Fork 0
Code Issues 24 Pull Requests Projects Releases 2 Wiki Activity

improved block list syncing, added forgotten stats.js file

This commit is contained in:
Kevin Froman 2019-01-15 23:57:47 -06:00
parent 0e6ab04996
commit 1ebed8d606
12 changed files with 100 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
onionr/api.py
View File

@ -109,7 +109,8 @@ class PublicAPI:
@app.route('/getblocklist')
def getBlockList():
bList = clientAPI._core.getBlockList()
dateAdjust = request.args.get('date')
bList = clientAPI._core.getBlockList(dateRec=dateAdjust)
for b in self.hideBlocks:
if b in bList:
bList.remove(b)
@ -304,7 +305,6 @@ class API:
@app.route('/queueResponseAdd/<name>', methods=['post'])
def queueResponseAdd(name):
print('added',name)
self.queueResponse[name] = request.form['data']
return Response('success')
@ -317,7 +317,6 @@ class API:
pass
else:
del self.queueResponse[name]
print(name, resp)
return Response(resp)
@app.route('/ping')
@ -380,6 +379,12 @@ class API:
except AttributeError:
pass
return Response("bye")
@app.route('/shutdownclean')
def shutdownClean():
# good for calling from other clients
self._core.daemonQueueAdd('shutdown')
return Response("bye")
@app.route('/getstats')
def getStats():

28
onionr/communicator2.py
View File

@ -21,7 +21,7 @@
'''
import sys, os, core, config, json, requests, time, logger, threading, base64, onionr, uuid
import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block
import onionrdaemontools, onionrsockets, onionrchat, onionr, onionrproofs
import onionrdaemontools, onionrsockets, onionr, onionrproofs, proofofmemory
import binascii
from dependencies import secrets
from defusedxml import minidom
@ -74,6 +74,9 @@ class OnionrCommunicatorDaemon:
# timestamp when the last online node was seen
self.lastNodeSeen = None
# Dict of time stamps for peer's block list lookup times, to avoid downloading full lists all the time
self.dbTimestamps = {}
# Clear the daemon queue for any dead messages
if os.path.exists(self._core.queueDB):
self._core.clearDaemonQueue()
@ -81,11 +84,12 @@ class OnionrCommunicatorDaemon:
# Loads in and starts the enabled plugins
plugins.reload()
self.proofofmemory = proofofmemory.ProofOfMemory(self)
# daemon tools are misc daemon functions, e.g. announce to online peers
# intended only for use by OnionrCommunicatorDaemon
self.daemonTools = onionrdaemontools.DaemonTools(self)
self._chat = onionrchat.OnionrChat(self)
if debug or developmentMode:
OnionrCommunicatorTimers(self, self.heartbeat, 30)
@ -164,7 +168,9 @@ class OnionrCommunicatorDaemon:
existingBlocks = self._core.getBlockList()
triedPeers = [] # list of peers we've tried this time around
maxBacklog = 1560 # Max amount of *new* block hashes to have already in queue, to avoid memory exhaustion
lastLookupTime = 0 # Last time we looked up a particular peer's list
for i in range(tryAmount):
listLookupCommand = 'getblocklist' # This is defined here to reset it each time
if len(self.blockQueue) >= maxBacklog:
break
if not self.isOnline:
@ -186,11 +192,21 @@ class OnionrCommunicatorDaemon:
triedPeers.append(peer)
if newDBHash != self._core.getAddressInfo(peer, 'DBHash'):
self._core.setAddressInfo(peer, 'DBHash', newDBHash)
# Get the last time we looked up a peer's stamp to only fetch blocks since then.
# Saved in memory only for privacy reasons
try:
lastLookupTime = self.dbTimestamps[peer]
except KeyError:
lastLookupTime = 0
else:
listLookupCommand += '?date=%s' % (lastLookupTime,)
try:
newBlocks = self.peerAction(peer, 'getblocklist') # get list of new block hashes
except Exception as error:
logger.warn('Could not get new blocks from %s.' % peer, error = error)
newBlocks = False
else:
self.dbTimestamps[peer] = self._core._utils.getRoundedEpoch(roundS=60)
if newBlocks != False:
# if request was a success
for i in newBlocks.split('\n'):
@ -224,8 +240,8 @@ class OnionrCommunicatorDaemon:
if self._core._utils.storageCounter.isFull():
break
self.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block
logger.info("Attempting to download %s..." % blockHash)
peerUsed = self.pickOnlinePeer()
logger.info("Attempting to download %s from %s..." % (blockHash[:12], peerUsed))
content = self.peerAction(peerUsed, 'getdata/' + blockHash) # block content from random peer (includes metadata)
if content != False and len(content) > 0:
try:
@ -247,7 +263,7 @@ class OnionrCommunicatorDaemon:
metadata = metas[0]
if self._core._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid, and verify nonce
if self._core._crypto.verifyPow(content): # check if POW is enough/correct
logger.info('Attempting to save block %s...' % blockHash)
logger.info('Attempting to save block %s...' % blockHash[:12])
try:
self._core.setData(content)
except onionrexceptions.DiskAllocationReached:
@ -403,6 +419,10 @@ class OnionrCommunicatorDaemon:
del self.connectTimes[peer]
except KeyError:
pass
try:
del self.dbTimestamps[peer]
except KeyError:
pass
try:
self.onlinePeers.remove(peer)
except ValueError:

22
onionr/core.py
View File

@ -21,7 +21,7 @@ import sqlite3, os, sys, time, math, base64, tarfile, nacl, logger, json, netcon
from onionrblockapi import Block
import onionrutils, onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions, onionrvalues
import onionrblacklist, onionrchat, onionrusers
import onionrblacklist, onionrusers
import dbcreator, onionrstorage, serializeddata
if sys.version_info < (3, 6):
@ -373,11 +373,11 @@ class Core:
try:
c.execute('INSERT INTO commands (command, data, date, responseID) VALUES(?, ?, ?, ?)', t)
conn.commit()
conn.close()
except sqlite3.OperationalError:
retData = False
self.daemonQueue()
events.event('queue_push', data = {'command': command, 'data': data}, onionr = None)
conn.close()
return retData
def daemonQueueGetResponse(self, responseID=''):
@ -602,24 +602,26 @@ class Core:
return
def getBlockList(self, unsaved = False): # TODO: Use unsaved??
def getBlockList(self, dateRec = None, unsaved = False):
'''
Get list of our blocks
'''
if dateRec == None:
dateRec = 0
conn = sqlite3.connect(self.blockDB, timeout=10)
c = conn.cursor()
if unsaved:
execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();'
else:
execute = 'SELECT hash FROM hashes ORDER BY dateReceived ASC;'
# if unsaved:
# execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();'
# else:
# execute = 'SELECT hash FROM hashes ORDER BY dateReceived ASC;'
execute = 'SELECT hash FROM hashes WHERE dateReceived >= ? ORDER BY dateReceived ASC;'
args = (dateRec,)
rows = list()
for row in c.execute(execute):
for row in c.execute(execute, args):
for i in row:
rows.append(i)
return rows
def getBlockDate(self, blockHash):

18
onionr/onionr.py
View File

@ -21,8 +21,9 @@
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import sys
if sys.version_info[0] == 2 or sys.version_info[1] < 5:
print('Error, Onionr requires Python 3.5+')
MIN_PY_VERSION = 6
if sys.version_info[0] == 2 or sys.version_info[1] < MIN_PY_VERSION:
print('Error, Onionr requires Python 3.%s+' % (MIN_PY_VERSION,))
sys.exit(1)
import os, base64, random, getpass, shutil, subprocess, requests, time, platform, datetime, re, json, getpass, sqlite3
import webbrowser, uuid, signal
@ -198,7 +199,6 @@ class Onionr:
'ui' : self.openUI,
'gui' : self.openUI,
'chat': self.startChat,
'getpassword': self.printWebPassword,
'get-password': self.printWebPassword,
@ -209,8 +209,6 @@ class Onionr:
'getpasswd': self.printWebPassword,
'get-passwd': self.printWebPassword,
'chat': self.startChat,
'friend': self.friendCmd,
'add-id': self.addID,
'change-id': self.changeID
@ -328,14 +326,6 @@ class Onionr:
else:
logger.error('Invalid key %s' % (key,))
def startChat(self):
try:
data = json.dumps({'peer': sys.argv[2], 'reason': 'chat'})
except IndexError:
logger.error('Must specify peer to chat with.')
else:
self.onionrCore.daemonQueueAdd('startSocket', data)
def getCommands(self):
return self.cmds
@ -811,7 +801,7 @@ class Onionr:
try:
# define stats messages here
totalBlocks = len(Block.getBlocks())
totalBlocks = len(self.onionrCore.getBlockList())
signedBlocks = len(Block.getBlocks(signed = True))
messages = {
# info about local client

50
onionr/onionrchat.py
View File

@ -1,50 +0,0 @@
'''
Onionr - P2P Anonymous Storage Network
Onionr Chat Messages
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import logger, time
class OnionrChat:
def __init__(self, communicatorInst):
'''OnionrChat uses onionrsockets (handled by the communicator) to exchange direct chat messages'''
self.communicator = communicatorInst
self._core = self.communicator._core
self._utils = self._core._utils
self.chats = {} # {'peer': {'date': date, message': message}}
self.chatSend = {}
def chatHandler(self):
while not self.communicator.shutdown:
for peer in self._core.socketServerConnData:
try:
assert self._core.socketReasons[peer] == "chat"
except (AssertionError, KeyError) as e:
logger.warn('Peer is not for chat')
continue
else:
self.chats[peer] = {'date': self._core.socketServerConnData[peer]['date'], 'data': self._core.socketServerConnData[peer]['data']}
logger.info("CHAT MESSAGE RECIEVED: %s" % self.chats[peer]['data'])
for peer in self.communicator.socketClient.sockets:
try:
logger.info(self.communicator.socketClient.connPool[peer]['data'])
self.communicator.socketClient.sendData(peer, "lol")
except:
pass
time.sleep(2)

1
onionr/onionrutils.py
View File

@ -183,7 +183,6 @@ class OnionrUtils:
if data != '':
data = '&data=' + urllib.parse.quote_plus(data)
payload = 'http://%s/%s%s' % (hostname, command, data)
print(payload,config.get('client.webpassword'))
try:
if post:
retData = requests.post(payload, data=postData, headers={'token': config.get('client.webpassword')}, timeout=(15, 30)).text

4
onionr/static-data/default_config.json
View File

@ -2,8 +2,8 @@
"general" : {
"dev_mode" : true,
"display_header" : false,
"minimum_block_pow": 3,
"minimum_send_pow": 3,
"minimum_block_pow": 1,
"minimum_send_pow": 1,
"socket_servers": false,
"security_level": 0,
"max_block_age": 2678400,

13
onionr/static-data/www/private/index.html
View File

@ -5,20 +5,27 @@
<title>
Onionr
</title>
<link rel='stylesheet' href='/shared/style/modal.css'>
<link rel='stylesheet' href='/shared/main/style.css'>
</head>
<body>
<div id="shutdownNotice" class='overlay'>
<div>
<p>Your node will shutdown. Thank you for using Onionr.</p>
</div>
</div>
<img class='logo' src='/shared/onionr-icon.png' alt='onionr logo'>
<span class='logoText'>Onionr Web Control Panel</span>
<div class='content'>
<a href='shutdown'>Shutdown node</a>
<button id='shutdownNode'>Shutdown Node</button>
<h2>Stats</h2>
<p>Uptime: <span id='uptime'></span></p>
<p>Stored Blocks: <span id='storedBlocks'></span></p>
<p>Connected nodes:</p>
<pre id='connectedNodes'></pre>
</div>
<script src='/shared/misc.js'></script>
<script src='/shared/main/stats.js'></script>
<script src='/shared/panel.js'></script>
</body>
<script src='/shared/misc.js'></script>
<script src='/shared/main/stats.js'></script>
</html>

5
onionr/static-data/www/shared/main/stats.js
View File

@ -17,14 +17,11 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>
*/
uptimeDisplay = document.getElementById('uptime')
connectedDisplay = document.getElementById('connectedNodes')
storedBlockDisplay = document.getElementById('storedBlocks')
pass = window.location.hash.replace('#', '')
stats = JSON.parse(httpGet('getstats', pass))
stats = JSON.parse(httpGet('getstats', webpass))
uptimeDisplay.innerText = stats['uptime'] + ' seconds'
connectedDisplay.innerText = stats['connectedNodes']
storedBlockDisplay.innerText = stats['blockCount']

14
onionr/static-data/www/shared/main/style.css
View File

@ -124,3 +124,17 @@ body{
display: block;
}
}
/*https://stackoverflow.com/a/16778646*/
.overlay {
visibility: hidden;
position: absolute;
left: 0px;
top: 0px;
width:100%;
opacity: 0.9;
height:100%;
text-align:center;
z-index: 1000;
background-color: black;
}

22
onionr/static-data/www/shared/misc.js
View File

@ -1,4 +1,16 @@
function httpGet(theUrl, webpass) {
webpass = document.location.hash.replace('#', '')
if (typeof webpass == "undefined"){
webpass = localStorage['webpass']
}
else{
localStorage['webpass'] = webpass
document.location.hash = ''
}
if (typeof webpass == "undefined" || webpass == ""){
alert('Web password was not found in memory or URL')
}
function httpGet(theUrl) {
var xmlHttp = new XMLHttpRequest()
xmlHttp.open( "GET", theUrl, false ) // false for synchronous request
xmlHttp.setRequestHeader('token', webpass)
@ -7,6 +19,10 @@ function httpGet(theUrl, webpass) {
return xmlHttp.responseText
}
else{
return "";
return ""
}
}
}
function overlay(overlayID) {
el = document.getElementById(overlayID)
el.style.visibility = (el.style.visibility == "visible") ? "hidden" : "visible"
}

6
onionr/static-data/www/shared/panel.js Normal file
View File

@ -0,0 +1,6 @@
shutdownBtn = document.getElementById('shutdownNode')
shutdownBtn.onclick = function(){
httpGet('shutdownclean')
overlay('shutdownNotice')
}