kev/Onionr
1
0
Fork 0
Code Issues 24 Pull Requests Projects Releases 2 Wiki Activity

fixes in board, refactoring core databases to not use core anymore

Browse Source
This commit is contained in:
Kevin Froman 2019-07-17 11:25:29 -05:00
parent 1ced21f40c
commit 000538ddc8
17 changed files with 43 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
onionr/communicatorutils/downloadblocks/shoulddownload.py
View File

@ -17,9 +17,10 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
from coredb import blockmetadb
def should_download(comm_inst, block_hash):
ret_data = True
if block_hash in comm_inst._core.getBlockList(): # Dont download block we have
if block_hash in blockmetadb.get_block_list(): # Dont download block we have
ret_data = False
else:
if comm_inst._core._blacklist.inBlacklist(block_hash): # Dont download blacklisted block

3
onionr/communicatorutils/housekeeping.py
View File

@ -21,6 +21,7 @@ import sqlite3
import logger
from onionrusers import onionrusers
from onionrutils import epoch
from coredb import blockmetadb
def clean_old_blocks(comm_inst):
'''Delete old blocks if our disk allocation is full/near full, and also expired blocks'''
@ -31,7 +32,7 @@ def clean_old_blocks(comm_inst):
logger.info('Deleted block: %s' % (bHash,))
while comm_inst._core.storage_counter.isFull():
oldest = comm_inst._core.getBlockList()[0]
oldest = blockmetadb.get_block_list()[0]
comm_inst._core._blacklist.addToDB(oldest)
comm_inst._core.removeBlock(oldest)
logger.info('Deleted block: %s' % (oldest,))

4
onionr/communicatorutils/lookupblocks.py
View File

@ -20,12 +20,12 @@
import logger, onionrproofs
from onionrutils import stringvalidators, epoch
from communicator import peeraction, onlinepeers
from coredb import blockmetadb
def lookup_blocks_from_communicator(comm_inst):
logger.info('Looking up new blocks...')
tryAmount = 2
newBlocks = ''
existingBlocks = comm_inst._core.getBlockList()
existingBlocks = blockmetadb.get_block_list()
triedPeers = [] # list of peers we've tried this time around
maxBacklog = 1560 # Max amount of *new* block hashes to have already in queue, to avoid memory exhaustion
lastLookupTime = 0 # Last time we looked up a particular peer's list

4
onionr/communicatorutils/servicecreator.py
View File

@ -19,14 +19,14 @@
'''
import communicator, onionrblockapi
from onionrutils import stringvalidators, bytesconverter
from coredb import blockmetadb
def service_creator(daemon):
assert isinstance(daemon, communicator.OnionrCommunicatorDaemon)
core = daemon._core
# Find socket connection blocks
# TODO cache blocks and only look at recently received ones
con_blocks = core.getBlocksByType('con')
con_blocks = blockmetadb.get_blocks_by_type('con')
for b in con_blocks:
if not b in daemon.active_services:
bl = onionrblockapi.Block(b, core=core, decrypt=True)

18
onionr/core.py
View File

@ -267,24 +267,6 @@ class Core:
'''
return coredb.keydb.transportinfo.set_address_info(self, address, key, data)
def getBlockList(self, dateRec = None, unsaved = False):
'''
Get list of our blocks
'''
return coredb.blockmetadb.get_block_list(self, dateRec, unsaved)
def getBlockDate(self, blockHash):
'''
Returns the date a block was received
'''
return coredb.blockmetadb.get_block_date(self, blockHash)
def getBlocksByType(self, blockType, orderDate=True):
'''
Returns a list of blocks by the type
'''
return coredb.blockmetadb.get_blocks_by_type(self, blockType, orderDate)
def getExpiredBlocks(self):
'''Returns a list of expired blocks'''
return coredb.blockmetadb.expiredblocks.get_expired_blocks(self)

13
onionr/coredb/blockmetadb/__init__.py
View File

@ -19,14 +19,15 @@
'''
import sqlite3
from . import expiredblocks, updateblockinfo, add
def get_block_list(core_inst, dateRec = None, unsaved = False):
from .. import dbfiles
def get_block_list(dateRec = None, unsaved = False):
'''
Get list of our blocks
'''
if dateRec == None:
dateRec = 0
conn = sqlite3.connect(core_inst.blockDB, timeout=30)
conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30)
c = conn.cursor()
execute = 'SELECT hash FROM hashes WHERE dateReceived >= ? ORDER BY dateReceived ASC;'
@ -38,12 +39,12 @@ def get_block_list(core_inst, dateRec = None, unsaved = False):
conn.close()
return rows
def get_block_date(core_inst, blockHash):
def get_block_date(blockHash):
'''
Returns the date a block was received
'''
conn = sqlite3.connect(core_inst.blockDB, timeout=30)
conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30)
c = conn.cursor()
execute = 'SELECT dateReceived FROM hashes WHERE hash=?;'
@ -54,12 +55,12 @@ def get_block_date(core_inst, blockHash):
conn.close()
return None
def get_blocks_by_type(core_inst, blockType, orderDate=True):
def get_blocks_by_type(blockType, orderDate=True):
'''
Returns a list of blocks by the type
'''
conn = sqlite3.connect(core_inst.blockDB, timeout=30)
conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30)
c = conn.cursor()
if orderDate:

5
onionr/coredb/dbfiles.py Normal file
View File

@ -0,0 +1,5 @@
from utils import identifyhome
home = identifyhome.identify_home()
if not home.endswith('/'): home += '/'
block_meta_db = '%sblock-metadata.db'

7
onionr/httpapi/miscclientapi/getblocks.py
View File

@ -21,6 +21,7 @@ from flask import Blueprint, Response, abort
import core, onionrblockapi
from httpapi import apiutils
from onionrutils import stringvalidators
from coredb import blockmetadb
c = core.Core()
@ -29,8 +30,8 @@ client_get_block = apiutils.GetBlockData(c)
client_get_blocks = Blueprint('miscclient', __name__)
@client_get_blocks.route('/getblocksbytype/<name>')
def getBlocksByType(name):
blocks = c.getBlocksByType(name)
def get_blocks_by_type_endpoint(name):
blocks = blockmetadb.get_blocks_by_type(name)
return Response(','.join(blocks))
@client_get_blocks.route('/getblockbody/<name>')
@ -49,7 +50,7 @@ def getBlockBodyData(name):
def getData(name):
resp = ""
if stringvalidators.validate_hash(name):
if name in c.getBlockList():
if name in blockmetadb.get_block_list():
try:
resp = client_get_block.get_block_data(name, decrypt=True)
except ValueError:

5
onionr/httpapi/miscpublicapi/getblocks.py
View File

@ -20,11 +20,12 @@
from flask import Response, abort
import config
from onionrutils import bytesconverter, stringvalidators
from coredb import blockmetadb
def get_public_block_list(clientAPI, publicAPI, request):
# Provide a list of our blocks, with a date offset
dateAdjust = request.args.get('date')
bList = clientAPI._core.getBlockList(dateRec=dateAdjust)
bList = blockmetadb.get_block_list(dateRec=dateAdjust)
if clientAPI._core.config.get('general.hide_created_blocks', True):
for b in publicAPI.hideBlocks:
if b in bList:
@ -37,7 +38,7 @@ def get_block_data(clientAPI, publicAPI, data):
resp = ''
if stringvalidators.validate_hash(data):
if not clientAPI._core.config.get('general.hide_created_blocks', True) or data not in publicAPI.hideBlocks:
if data in clientAPI._core.getBlockList():
if data in blockmetadb.get_block_list():
block = clientAPI.getBlockData(data, raw=True)
try:
block = block.encode() # Encode in case data is binary

8
onionr/onionrblockapi.py
View File

@ -22,7 +22,7 @@ import core as onionrcore, logger, config, onionrexceptions, nacl.exceptions
import json, os, sys, datetime, base64, onionrstorage
from onionrusers import onionrusers
from onionrutils import stringvalidators, epoch
from coredb import blockmetadb
class Block:
blockCacheOrder = list() # NEVER write your own code that writes to this!
blockCache = dict() # should never be accessed directly, look at Block.getCache()
@ -89,7 +89,7 @@ class Block:
# Check for replay attacks
try:
if epoch.get_epoch() - self.core.getBlockDate(self.hash) > 60:
if epoch.get_epoch() - blockmetadb.get_block_date(self.hash) > 60:
assert self.core._crypto.replayTimestampValidation(self.bmetadata['rply'])
except (AssertionError, KeyError, TypeError) as e:
if not self.bypassReplayCheck:
@ -180,7 +180,7 @@ class Block:
self.signature = self.getHeader('sig', None)
# signed data is jsonMeta + block content (no linebreak)
self.signedData = (None if not self.isSigned() else self.getHeader('meta') + self.getContent())
self.date = self.getCore().getBlockDate(self.getHash())
self.date = blockmetadb.get_block_date(self.getHash())
self.claimedTime = self.getHeader('time', None)
if not self.getDate() is None:
@ -541,7 +541,7 @@ class Block:
parent = Block(hash = parent, core = core)
relevant_blocks = list()
blocks = (core.getBlockList() if type is None else core.getBlocksByType(type))
blocks = (blockmetadb.get_block_list() if type is None else blockmetadb.get_blocks_by_type(type))
for block in blocks:
if Block.exists(block):

4
onionr/onionrcommands/onionrstatistics.py
View File

@ -23,11 +23,11 @@ from onionrblockapi import Block
import onionr
from onionrutils import checkcommunicator, mnemonickeys
from utils import sizeutils
from coredb import blockmetadb
def show_stats(o_inst):
try:
# define stats messages here
totalBlocks = len(o_inst.onionrCore.getBlockList())
totalBlocks = len(blockmetadb.get_block_list())
signedBlocks = len(Block.getBlocks(signed = True))
messages = {
# info about local client

3
onionr/onionrutils/importnewblocks.py
View File

@ -20,13 +20,14 @@
import glob
import logger, core
from onionrutils import blockmetadata
from coredb import blockmetadb
def import_new_blocks(core_inst=None, scanDir=''):
'''
This function is intended to scan for new blocks ON THE DISK and import them
'''
if core_inst is None:
core_inst = core.Core()
blockList = core_inst.getBlockList()
blockList = blockmetadb.get_block_list()
exist = False
if scanDir == '':
scanDir = core_inst.blockDataLocation

4
onionr/serializeddata.py
View File

@ -19,7 +19,7 @@
'''
import core, json
from coredb import blockmetadb
class SerializedData:
def __init__(self, coreInst):
'''
@ -38,6 +38,6 @@ class SerializedData:
stats = {}
stats['uptime'] = self._core.onionrInst.communicatorInst.getUptime()
stats['connectedNodes'] = '\n'.join(self._core.onionrInst.communicatorInst.onlinePeers)
stats['blockCount'] = len(self._core.getBlockList())
stats['blockCount'] = len(blockmetadb.get_block_list())
stats['blockQueueCount'] = len(self._core.onionrInst.communicatorInst.blockQueue)
return json.dumps(stats)

4
onionr/static-data/default-plugins/flow/main.py
View File

@ -24,7 +24,7 @@ from onionrblockapi import Block
import logger, config
from onionrutils import escapeansi, epoch
locale.setlocale(locale.LC_ALL, '')
from coredb import blockmetadb
sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
import flowapi # import after path insert
flask_blueprint = flowapi.flask_blueprint
@ -73,7 +73,7 @@ class OnionrFlow:
time.sleep(1)
try:
while self.flowRunning:
for block in self.myCore.getBlocksByType('txt'):
for block in blockmetadb.get_blocks_by_type('txt'):
block = Block(block)
if block.getMetadata('ch') != self.channel:
continue

3
onionr/static-data/default-plugins/pms/loadinbox.py
View File

@ -18,13 +18,14 @@
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import onionrblockapi
from coredb import blockmetadb
def load_inbox(myCore):
inbox_list = []
deleted = myCore.keyStore.get('deleted_mail')
if deleted is None:
deleted = []
for blockHash in myCore.getBlocksByType('pm'):
for blockHash in blockmetadb.get_blocks_by_type('pm'):
block = onionrblockapi.Block(blockHash, core=myCore)
block.decrypt()
if block.decrypted and blockHash not in deleted:

3
onionr/static-data/default-plugins/pms/main.py
View File

@ -25,6 +25,7 @@ import onionrexceptions
from onionrusers import onionrusers
from onionrutils import stringvalidators, escapeansi, bytesconverter
import locale, sys, os, json
from coredb import blockmetadb
locale.setlocale(locale.LC_ALL, '')
@ -80,7 +81,7 @@ class OnionrMail:
subject = ''
# this could use a lot of memory if someone has received a lot of messages
for blockHash in self.myCore.getBlocksByType('pm'):
for blockHash in blockmetadb.get_blocks_by_type('pm'):
pmBlocks[blockHash] = Block(blockHash, core=self.myCore)
pmBlocks[blockHash].decrypt()
blockCount = 0

19
onionr/static-data/www/board/index.html
View File

@ -55,25 +55,6 @@
Anonymous message board
</h2>
</div>
<div class="column is-7">
<div class="field">
<label class="label">Open Site</label>
<div class="field has-addons">
<p class="control">
<a class="button is-static">Identity</a>
</p>
<p class="control is-expanded">
<input id="myPub" class="input myPub" type="text" readonly>
</p>
<p class="control">
<a id="myPubCopy" class="button is-primary">Copy</a>
</p>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</section>