2019-09-23 23:20:14 +00:00
<!doctype html>
< html lang = "en" >
< head >
< meta charset = "utf-8" >
< meta name = "viewport" content = "width=device-width, initial-scale=1, minimum-scale=1" / >
2020-02-04 20:46:17 +00:00
< meta name = "generator" content = "pdoc 0.7.4" / >
< title > src.onionrstorage API documentation< / title >
2019-09-23 23:20:14 +00:00
< meta name = "description" content = "Onionr - Private P2P Communication …" / >
< link href = 'https://cdnjs.cloudflare.com/ajax/libs/normalize/8.0.0/normalize.min.css' rel = 'stylesheet' >
< link href = 'https://cdnjs.cloudflare.com/ajax/libs/10up-sanitize.css/8.0.0/sanitize.min.css' rel = 'stylesheet' >
< link href = "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css" rel = "stylesheet" >
2020-02-04 20:46:17 +00:00
< style > . flex { display : flex !important } body { line-height : 1.5 em } # content { padding : 20 px } # sidebar { padding : 30 px ; overflow : hidden } . http-server-breadcrumbs { font-size : 130 % ; margin : 0 0 15 px 0 } # footer { font-size : .75 em ; padding : 5 px 30 px ; border-top : 1 px solid #ddd ; text-align : right } # footer p { margin : 0 0 0 1 em ; display : inline-block } # footer p : last-child { margin-right : 30 px } h1 , h2 , h3 , h4 , h5 { font-weight : 300 } h1 { font-size : 2.5 em ; line-height : 1.1 em } h2 { font-size : 1.75 em ; margin : 1 em 0 .50 em 0 } h3 { font-size : 1.4 em ; margin : 25 px 0 10 px 0 } h4 { margin : 0 ; font-size : 105 % } a { color : #058 ; text-decoration : none ; transition : color .3 s ease-in-out } a : hover { color : #e82 } . title code { font-weight : bold } h2 [ id ^ = "header-" ] { margin-top : 2 em } . ident { color : #900 } pre code { background : #f8f8f8 ; font-size : .8 em ; line-height : 1.4 em } code { background : #f2f2f1 ; padding : 1 px 4 px ; overflow-wrap : break-word } h1 code { background : transparent } pre { background : #f8f8f8 ; border : 0 ; border-top : 1 px solid #ccc ; border-bottom : 1 px solid #ccc ; margin : 1 em 0 ; padding : 1 ex } # http-server-module-list { display : flex ; flex-flow : column } # http-server-module-list div { display : flex } # http-server-module-list dt { min-width : 10 % } # http-server-module-list p { margin-top : 0 } . toc ul , # index { list-style-type : none ; margin : 0 ; padding : 0 } # index code { background : transparent } # index h3 { border-bottom : 1 px solid #ddd } # index ul { padding : 0 } # index h4 { font-weight : bold } # index h4 + ul { margin-bottom : .6 em } @ media ( min-width : 200ex ) { # index . two-column { column-count : 2 } } @ media ( min-width : 300ex ) { # index . two-column { column-count : 3 } } dl { margin-bottom : 2 em } dl dl : last-child { margin-bottom : 4 em } dd { margin : 0 0 1 em 3 em } # header-classes + dl > dd { margin-bottom : 3 em } dd dd { margin-left : 2 em } dd p { margin : 10 px 0 } . name { background : #eee ; font-weight : bold ; font-size : .85 em ; padding : 5 px 10 px ; display : inline-block ; min-width : 40 % } . name : hover { background : #e0e0e0 } . name > span : first-child { white-space : nowrap } . name . class > span : nth-child ( 2 ) { margin-left : .4 em } . inherited { color : #999 ; border-left : 5 px solid #eee ; padding-left : 1 em } . inheritance em { font-style : normal ; font-weight : bold } . desc h2 { font-weight : 400 ; font-size : 1.25 em } . desc h3 { font-size : 1 em } . desc dt code { background : inherit } . source summary , . git-link-div { color : #666 ; text-align : right ; font-weight : 400 ; font-size : .8 em ; text-transform : uppercase } . source summary > * { white-space : nowrap ; cursor : pointer } . git-link { color : inherit ; margin-left : 1 em } . source pre { max-height : 500 px ; overflow : auto ; margin : 0 } . source pre code { font-size : 12 px ; overflow : visible } . hlist { list-style : none } . hlist li { display : inline } . hlist li : after { content : ',\2002' } . hlist li : last-child : after { content : none } . hlist . hlist { display : inline ; padding-left : 1 em } img { max-width : 100 % } . admonition { padding : .1 em .5 em ; margin-bottom : 1 em } . admonition-title { font-weight : bold } . admonition . note , . admonition . info , . admonition . important { background : #aef } . admonition . todo , . admonition . versionadded , . admonition . tip , . admonition . hint { background : #dfd } . admonition . warning , . admonition . versionchanged , . admonition . deprecated { background : #fd4 } . admonition . error , . admonition . danger , . admonition . caution { background : lightpink } < / style >
2019-09-23 23:20:14 +00:00
< style media = "screen and (min-width: 700px)" > @ media screen and ( min-width : 700px ) { # sidebar { width : 30 % } # content { width : 70 % ; max-width : 100 ch ; padding : 3 em 4 em ; border-left : 1 px solid #ddd } pre code { font-size : 1 em } . item . name { font-size : 1 em } main { display : flex ; flex-direction : row-reverse ; justify-content : flex-end } . toc ul ul , # index ul { padding-left : 1.5 em } . toc > ul > li { margin-top : .5 em } } < / style >
< style media = "print" > @ media print { # sidebar h1 { page-break-before : always } . source { display : none } } @ media print { * { background : transparent !important ; color : #000 !important ; box-shadow : none !important ; text-shadow : none !important } a [ href ] : after { content : " (" attr ( href ) ")" ; font-size : 90 % } a [ href ] [ title ] : after { content : none } abbr [ title ] : after { content : " (" attr ( title ) ")" } . ir a : after , a [ href ^ = "javascript:" ] : after , a [ href ^ = "#" ] : after { content : "" } pre , blockquote { border : 1 px solid #999 ; page-break-inside : avoid } thead { display : table-header-group } tr , img { page-break-inside : avoid } img { max-width : 100 % !important } @ page { margin : 0 . 5cm } p , h2 , h3 { orphans : 3 ; widows : 3 } h1 , h2 , h3 , h4 , h5 , h6 { page-break-after : avoid } } < / style >
< / head >
< body >
< main >
< article id = "content" >
< header >
2020-02-04 20:46:17 +00:00
< h1 class = "title" > Module < code > src.onionrstorage< / code > < / h1 >
2019-09-23 23:20:14 +00:00
< / header >
< section id = "section-intro" >
< p > Onionr - Private P2P Communication< / p >
< p > This file handles block storage, providing an abstraction for storing blocks between file system and database< / p >
< details class = "source" >
2020-02-04 20:46:17 +00:00
< summary >
< span > Expand source code< / span >
< / summary >
2019-09-23 23:20:14 +00:00
< pre > < code class = "python" > ' ' '
Onionr - Private P2P Communication
This file handles block storage, providing an abstraction for storing blocks between file system and database
' ' '
2020-02-04 20:46:17 +00:00
import sys
import sqlite3
import os
from onionrutils import bytesconverter
from onionrutils import stringvalidators
from coredb import dbfiles
import filepaths
import onionrcrypto
import onionrexceptions
from onionrsetup import dbcreator
from onionrcrypto import hashers
from . import setdata
2019-09-23 23:20:14 +00:00
' ' '
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see < https://www.gnu.org/licenses/> .
' ' '
2020-02-04 20:46:17 +00:00
DB_ENTRY_SIZE_LIMIT = 10000 # Will be a config option
2019-09-23 23:20:14 +00:00
set_data = setdata.set_data
2020-02-04 20:46:17 +00:00
2019-09-23 23:20:14 +00:00
def _dbInsert(blockHash, data):
conn = sqlite3.connect(dbfiles.block_data_db, timeout=10)
c = conn.cursor()
data = (blockHash, data)
c.execute(' INSERT INTO blockData (hash, data) VALUES(?, ?);' , data)
conn.commit()
conn.close()
2020-02-04 20:46:17 +00:00
2019-09-23 23:20:14 +00:00
def _dbFetch(blockHash):
conn = sqlite3.connect(dbfiles.block_data_db, timeout=10)
c = conn.cursor()
for i in c.execute(' SELECT data from blockData where hash = ?' , (blockHash,)):
return i[0]
conn.commit()
conn.close()
return None
2020-02-04 20:46:17 +00:00
2019-09-23 23:20:14 +00:00
def deleteBlock(blockHash):
2020-02-04 20:46:17 +00:00
# You should call removeblock.remove_block if you automatically want to remove storage byte count
2019-09-23 23:20:14 +00:00
if os.path.exists(' %s/%s.dat' % (filepaths.block_data_location, blockHash)):
os.remove(' %s/%s.dat' % (filepaths.block_data_location, blockHash))
return True
conn = sqlite3.connect(dbfiles.block_data_db, timeout=10)
c = conn.cursor()
data = (blockHash,)
c.execute(' DELETE FROM blockData where hash = ?' , data)
conn.commit()
conn.close()
return True
2020-02-04 20:46:17 +00:00
2019-09-23 23:20:14 +00:00
def store(data, blockHash=' ' ):
if not stringvalidators.validate_hash(blockHash): raise ValueError
ourHash = hashers.sha3_hash(data)
if blockHash != ' ' :
2020-02-04 20:46:17 +00:00
if not ourHash == blockHash:
raise ValueError(' Hash specified does not meet internal hash check' )
2019-09-23 23:20:14 +00:00
else:
blockHash = ourHash
if DB_ENTRY_SIZE_LIMIT > = sys.getsizeof(data):
_dbInsert(blockHash, data)
else:
with open(' %s/%s.dat' % (filepaths.block_data_location, blockHash), ' wb' ) as blockFile:
blockFile.write(data)
2020-02-04 20:46:17 +00:00
2019-09-23 23:20:14 +00:00
def getData(bHash):
if not stringvalidators.validate_hash(bHash): raise ValueError
bHash = bytesconverter.bytes_to_str(bHash)
# First check DB for data entry by hash
# if no entry, check disk
# If no entry in either, raise an exception
retData = None
fileLocation = ' %s/%s.dat' % (filepaths.block_data_location, bHash)
2020-02-04 20:46:17 +00:00
not_found_msg = " Flock data not found for: "
2019-09-23 23:20:14 +00:00
if os.path.exists(fileLocation):
with open(fileLocation, ' rb' ) as block:
retData = block.read()
else:
retData = _dbFetch(bHash)
if retData is None:
2020-02-04 20:46:17 +00:00
raise onionrexceptions.NoDataAvailable(not_found_msg + str(bHash))
2019-09-23 23:20:14 +00:00
return retData< / code > < / pre >
< / details >
< / section >
< section >
< h2 class = "section-title" id = "header-submodules" > Sub-modules< / h2 >
< dl >
2020-02-04 20:46:17 +00:00
< dt > < code class = "name" > < a title = "src.onionrstorage.removeblock" href = "removeblock.html" > src.onionrstorage.removeblock< / a > < / code > < / dt >
2019-09-23 23:20:14 +00:00
< dd >
2020-02-04 20:46:17 +00:00
< section class = "desc" > < p > Onionr - Private P2P Communication …< / p > < / section >
2019-09-23 23:20:14 +00:00
< / dd >
2020-02-04 20:46:17 +00:00
< dt > < code class = "name" > < a title = "src.onionrstorage.setdata" href = "setdata.html" > src.onionrstorage.setdata< / a > < / code > < / dt >
2019-09-23 23:20:14 +00:00
< dd >
< section class = "desc" > < / section >
< / dd >
< / dl >
< / section >
< section >
< / section >
< section >
< h2 class = "section-title" id = "header-functions" > Functions< / h2 >
< dl >
2020-02-04 20:46:17 +00:00
< dt id = "src.onionrstorage.deleteBlock" > < code class = "name flex" >
2019-09-23 23:20:14 +00:00
< span > def < span class = "ident" > deleteBlock< / span > < / span > (< span > blockHash)< / span >
< / code > < / dt >
< dd >
< section class = "desc" > < / section >
< details class = "source" >
2020-02-04 20:46:17 +00:00
< summary >
< span > Expand source code< / span >
< / summary >
2019-09-23 23:20:14 +00:00
< pre > < code class = "python" > def deleteBlock(blockHash):
2020-02-04 20:46:17 +00:00
# You should call removeblock.remove_block if you automatically want to remove storage byte count
2019-09-23 23:20:14 +00:00
if os.path.exists(' %s/%s.dat' % (filepaths.block_data_location, blockHash)):
os.remove(' %s/%s.dat' % (filepaths.block_data_location, blockHash))
return True
conn = sqlite3.connect(dbfiles.block_data_db, timeout=10)
c = conn.cursor()
data = (blockHash,)
c.execute(' DELETE FROM blockData where hash = ?' , data)
conn.commit()
conn.close()
return True< / code > < / pre >
< / details >
< / dd >
2020-02-04 20:46:17 +00:00
< dt id = "src.onionrstorage.getData" > < code class = "name flex" >
2019-09-23 23:20:14 +00:00
< span > def < span class = "ident" > getData< / span > < / span > (< span > bHash)< / span >
< / code > < / dt >
< dd >
< section class = "desc" > < / section >
< details class = "source" >
2020-02-04 20:46:17 +00:00
< summary >
< span > Expand source code< / span >
< / summary >
2019-09-23 23:20:14 +00:00
< pre > < code class = "python" > def getData(bHash):
if not stringvalidators.validate_hash(bHash): raise ValueError
bHash = bytesconverter.bytes_to_str(bHash)
# First check DB for data entry by hash
# if no entry, check disk
# If no entry in either, raise an exception
retData = None
fileLocation = ' %s/%s.dat' % (filepaths.block_data_location, bHash)
2020-02-04 20:46:17 +00:00
not_found_msg = " Flock data not found for: "
2019-09-23 23:20:14 +00:00
if os.path.exists(fileLocation):
with open(fileLocation, ' rb' ) as block:
retData = block.read()
else:
retData = _dbFetch(bHash)
if retData is None:
2020-02-04 20:46:17 +00:00
raise onionrexceptions.NoDataAvailable(not_found_msg + str(bHash))
2019-09-23 23:20:14 +00:00
return retData< / code > < / pre >
< / details >
< / dd >
2020-02-04 20:46:17 +00:00
< dt id = "src.onionrstorage.store" > < code class = "name flex" >
2019-09-23 23:20:14 +00:00
< span > def < span class = "ident" > store< / span > < / span > (< span > data, blockHash='')< / span >
< / code > < / dt >
< dd >
< section class = "desc" > < / section >
< details class = "source" >
2020-02-04 20:46:17 +00:00
< summary >
< span > Expand source code< / span >
< / summary >
2019-09-23 23:20:14 +00:00
< pre > < code class = "python" > def store(data, blockHash=' ' ):
if not stringvalidators.validate_hash(blockHash): raise ValueError
ourHash = hashers.sha3_hash(data)
if blockHash != ' ' :
2020-02-04 20:46:17 +00:00
if not ourHash == blockHash:
raise ValueError(' Hash specified does not meet internal hash check' )
2019-09-23 23:20:14 +00:00
else:
blockHash = ourHash
if DB_ENTRY_SIZE_LIMIT > = sys.getsizeof(data):
_dbInsert(blockHash, data)
else:
with open(' %s/%s.dat' % (filepaths.block_data_location, blockHash), ' wb' ) as blockFile:
blockFile.write(data)< / code > < / pre >
< / details >
< / dd >
< / dl >
< / section >
< section >
< / section >
< / article >
< nav id = "sidebar" >
< h1 > Index< / h1 >
< div class = "toc" >
< ul > < / ul >
< / div >
< ul id = "index" >
< li > < h3 > Super-module< / h3 >
< ul >
2020-02-04 20:46:17 +00:00
< li > < code > < a title = "src" href = "../index.html" > src< / a > < / code > < / li >
2019-09-23 23:20:14 +00:00
< / ul >
< / li >
< li > < h3 > < a href = "#header-submodules" > Sub-modules< / a > < / h3 >
< ul >
2020-02-04 20:46:17 +00:00
< li > < code > < a title = "src.onionrstorage.removeblock" href = "removeblock.html" > src.onionrstorage.removeblock< / a > < / code > < / li >
< li > < code > < a title = "src.onionrstorage.setdata" href = "setdata.html" > src.onionrstorage.setdata< / a > < / code > < / li >
2019-09-23 23:20:14 +00:00
< / ul >
< / li >
< li > < h3 > < a href = "#header-functions" > Functions< / a > < / h3 >
< ul class = "" >
2020-02-04 20:46:17 +00:00
< li > < code > < a title = "src.onionrstorage.deleteBlock" href = "#src.onionrstorage.deleteBlock" > deleteBlock< / a > < / code > < / li >
< li > < code > < a title = "src.onionrstorage.getData" href = "#src.onionrstorage.getData" > getData< / a > < / code > < / li >
< li > < code > < a title = "src.onionrstorage.store" href = "#src.onionrstorage.store" > store< / a > < / code > < / li >
2019-09-23 23:20:14 +00:00
< / ul >
< / li >
< / ul >
< / nav >
< / main >
< footer id = "footer" >
2020-02-04 20:46:17 +00:00
< p > Generated by < a href = "https://pdoc3.github.io/pdoc" > < cite > pdoc< / cite > 0.7.4< / a > .< / p >
2019-09-23 23:20:14 +00:00
< / footer >
< script src = "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js" > < / script >
< script > hljs . initHighlightingOnLoad ( ) < / script >
< / body >
< / html >
