2019-09-23 23:20:14 +00:00
<!doctype html>
< html lang = "en" >
< head >
< meta charset = "utf-8" >
< meta name = "viewport" content = "width=device-width, initial-scale=1, minimum-scale=1" / >
2020-02-04 20:46:17 +00:00
< meta name = "generator" content = "pdoc 0.7.4" / >
< title > src.communicatorutils.downloadblocks API documentation< / title >
2019-09-23 23:20:14 +00:00
< meta name = "description" content = "Onionr - Private P2P Communication …" / >
< link href = 'https://cdnjs.cloudflare.com/ajax/libs/normalize/8.0.0/normalize.min.css' rel = 'stylesheet' >
< link href = 'https://cdnjs.cloudflare.com/ajax/libs/10up-sanitize.css/8.0.0/sanitize.min.css' rel = 'stylesheet' >
< link href = "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css" rel = "stylesheet" >
2020-02-04 20:46:17 +00:00
< style > . flex { display : flex !important } body { line-height : 1.5 em } # content { padding : 20 px } # sidebar { padding : 30 px ; overflow : hidden } . http-server-breadcrumbs { font-size : 130 % ; margin : 0 0 15 px 0 } # footer { font-size : .75 em ; padding : 5 px 30 px ; border-top : 1 px solid #ddd ; text-align : right } # footer p { margin : 0 0 0 1 em ; display : inline-block } # footer p : last-child { margin-right : 30 px } h1 , h2 , h3 , h4 , h5 { font-weight : 300 } h1 { font-size : 2.5 em ; line-height : 1.1 em } h2 { font-size : 1.75 em ; margin : 1 em 0 .50 em 0 } h3 { font-size : 1.4 em ; margin : 25 px 0 10 px 0 } h4 { margin : 0 ; font-size : 105 % } a { color : #058 ; text-decoration : none ; transition : color .3 s ease-in-out } a : hover { color : #e82 } . title code { font-weight : bold } h2 [ id ^ = "header-" ] { margin-top : 2 em } . ident { color : #900 } pre code { background : #f8f8f8 ; font-size : .8 em ; line-height : 1.4 em } code { background : #f2f2f1 ; padding : 1 px 4 px ; overflow-wrap : break-word } h1 code { background : transparent } pre { background : #f8f8f8 ; border : 0 ; border-top : 1 px solid #ccc ; border-bottom : 1 px solid #ccc ; margin : 1 em 0 ; padding : 1 ex } # http-server-module-list { display : flex ; flex-flow : column } # http-server-module-list div { display : flex } # http-server-module-list dt { min-width : 10 % } # http-server-module-list p { margin-top : 0 } . toc ul , # index { list-style-type : none ; margin : 0 ; padding : 0 } # index code { background : transparent } # index h3 { border-bottom : 1 px solid #ddd } # index ul { padding : 0 } # index h4 { font-weight : bold } # index h4 + ul { margin-bottom : .6 em } @ media ( min-width : 200ex ) { # index . two-column { column-count : 2 } } @ media ( min-width : 300ex ) { # index . two-column { column-count : 3 } } dl { margin-bottom : 2 em } dl dl : last-child { margin-bottom : 4 em } dd { margin : 0 0 1 em 3 em } # header-classes + dl > dd { margin-bottom : 3 em } dd dd { margin-left : 2 em } dd p { margin : 10 px 0 } . name { background : #eee ; font-weight : bold ; font-size : .85 em ; padding : 5 px 10 px ; display : inline-block ; min-width : 40 % } . name : hover { background : #e0e0e0 } . name > span : first-child { white-space : nowrap } . name . class > span : nth-child ( 2 ) { margin-left : .4 em } . inherited { color : #999 ; border-left : 5 px solid #eee ; padding-left : 1 em } . inheritance em { font-style : normal ; font-weight : bold } . desc h2 { font-weight : 400 ; font-size : 1.25 em } . desc h3 { font-size : 1 em } . desc dt code { background : inherit } . source summary , . git-link-div { color : #666 ; text-align : right ; font-weight : 400 ; font-size : .8 em ; text-transform : uppercase } . source summary > * { white-space : nowrap ; cursor : pointer } . git-link { color : inherit ; margin-left : 1 em } . source pre { max-height : 500 px ; overflow : auto ; margin : 0 } . source pre code { font-size : 12 px ; overflow : visible } . hlist { list-style : none } . hlist li { display : inline } . hlist li : after { content : ',\2002' } . hlist li : last-child : after { content : none } . hlist . hlist { display : inline ; padding-left : 1 em } img { max-width : 100 % } . admonition { padding : .1 em .5 em ; margin-bottom : 1 em } . admonition-title { font-weight : bold } . admonition . note , . admonition . info , . admonition . important { background : #aef } . admonition . todo , . admonition . versionadded , . admonition . tip , . admonition . hint { background : #dfd } . admonition . warning , . admonition . versionchanged , . admonition . deprecated { background : #fd4 } . admonition . error , . admonition . danger , . admonition . caution { background : lightpink } < / style >
2019-09-23 23:20:14 +00:00
< style media = "screen and (min-width: 700px)" > @ media screen and ( min-width : 700px ) { # sidebar { width : 30 % } # content { width : 70 % ; max-width : 100 ch ; padding : 3 em 4 em ; border-left : 1 px solid #ddd } pre code { font-size : 1 em } . item . name { font-size : 1 em } main { display : flex ; flex-direction : row-reverse ; justify-content : flex-end } . toc ul ul , # index ul { padding-left : 1.5 em } . toc > ul > li { margin-top : .5 em } } < / style >
< style media = "print" > @ media print { # sidebar h1 { page-break-before : always } . source { display : none } } @ media print { * { background : transparent !important ; color : #000 !important ; box-shadow : none !important ; text-shadow : none !important } a [ href ] : after { content : " (" attr ( href ) ")" ; font-size : 90 % } a [ href ] [ title ] : after { content : none } abbr [ title ] : after { content : " (" attr ( title ) ")" } . ir a : after , a [ href ^ = "javascript:" ] : after , a [ href ^ = "#" ] : after { content : "" } pre , blockquote { border : 1 px solid #999 ; page-break-inside : avoid } thead { display : table-header-group } tr , img { page-break-inside : avoid } img { max-width : 100 % !important } @ page { margin : 0 . 5cm } p , h2 , h3 { orphans : 3 ; widows : 3 } h1 , h2 , h3 , h4 , h5 , h6 { page-break-after : avoid } } < / style >
< / head >
< body >
< main >
< article id = "content" >
< header >
2020-02-04 20:46:17 +00:00
< h1 class = "title" > Module < code > src.communicatorutils.downloadblocks< / code > < / h1 >
2019-09-23 23:20:14 +00:00
< / header >
< section id = "section-intro" >
< p > Onionr - Private P2P Communication< / p >
< p > Download blocks using the communicator instance< / p >
< details class = "source" >
2020-02-04 20:46:17 +00:00
< summary >
< span > Expand source code< / span >
< / summary >
< pre > < code class = "python" > " " "
2019-09-23 23:20:14 +00:00
Onionr - Private P2P Communication
Download blocks using the communicator instance
2020-02-04 20:46:17 +00:00
" " "
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from communicator import OnionrCommunicatorDaemon
from gevent import spawn
import onionrexceptions
import logger
import onionrpeers
import communicator
from communicator import peeraction
from communicator import onlinepeers
from onionrutils import blockmetadata
from onionrutils import validatemetadata
from coredb import blockmetadb
from onionrutils.localcommand import local_command
import onionrcrypto
import onionrstorage
from onionrblocks import onionrblacklist
from onionrblocks import storagecounter
from . import shoulddownload
" " "
2019-09-23 23:20:14 +00:00
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see < https://www.gnu.org/licenses/> .
2020-02-04 20:46:17 +00:00
" " "
def download_blocks_from_communicator(comm_inst: " OnionrCommunicatorDaemon" ):
" " " Use communicator instance to download blocks in the comms' s queue" " "
2019-09-23 23:20:14 +00:00
blacklist = onionrblacklist.OnionrBlackList()
storage_counter = storagecounter.StorageCounter()
LOG_SKIP_COUNT = 50 # for how many iterations we skip logging the counter
count: int = 0
metadata_validation_result: bool = False
# Iterate the block queue in the communicator
for blockHash in list(comm_inst.blockQueue):
count += 1
2020-02-04 20:46:17 +00:00
2019-09-23 23:20:14 +00:00
triedQueuePeers = [] # List of peers we' ve tried for a block
try:
blockPeers = list(comm_inst.blockQueue[blockHash])
except KeyError:
blockPeers = []
removeFromQueue = True
if not shoulddownload.should_download(comm_inst, blockHash):
continue
if comm_inst.shutdown or not comm_inst.isOnline or storage_counter.is_full():
# Exit loop if shutting down or offline, or disk allocation reached
break
# Do not download blocks being downloaded
if blockHash in comm_inst.currentDownloading:
continue
2020-02-04 20:46:17 +00:00
if len(comm_inst.onlinePeers) == 0:
break
2019-09-23 23:20:14 +00:00
comm_inst.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block
if len(blockPeers) == 0:
2020-02-04 20:46:17 +00:00
try:
peerUsed = onlinepeers.pick_online_peer(comm_inst)
except onionrexceptions.OnlinePeerNeeded:
continue
2019-09-23 23:20:14 +00:00
else:
blockPeers = onionrcrypto.cryptoutils.random_shuffle(blockPeers)
peerUsed = blockPeers.pop(0)
if not comm_inst.shutdown and peerUsed.strip() != ' ' :
logger.info(" Attempting to download %s from %s..." % (blockHash[:12], peerUsed))
content = peeraction.peer_action(comm_inst, peerUsed, ' getdata/' + blockHash, max_resp_size=3000000) # block content from random peer (includes metadata)
2020-02-04 20:46:17 +00:00
if content is not False and len(content) > 0:
2019-09-23 23:20:14 +00:00
try:
content = content.encode()
except AttributeError:
pass
realHash = onionrcrypto.hashers.sha3_hash(content)
try:
realHash = realHash.decode() # bytes on some versions for some reason
except AttributeError:
pass
if realHash == blockHash:
#content = content.decode() # decode here because sha3Hash needs bytes above
metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
metadata = metas[0]
try:
2020-02-04 20:46:17 +00:00
metadata_validation_result = \
validatemetadata.validate_metadata(metadata, metas[2])
2019-09-23 23:20:14 +00:00
except onionrexceptions.DataExists:
metadata_validation_result = False
if metadata_validation_result: # check if metadata is valid, and verify nonce
if onionrcrypto.cryptoutils.verify_POW(content): # check if POW is enough/correct
logger.info(' Attempting to save block %s...' % blockHash[:12])
try:
onionrstorage.set_data(content)
except onionrexceptions.DataExists:
logger.warn(' Data is already set for %s ' % (blockHash,))
except onionrexceptions.DiskAllocationReached:
logger.error(' Reached disk allocation allowance, cannot save block %s.' % (blockHash,))
removeFromQueue = False
else:
blockmetadb.add_to_block_DB(blockHash, dataSaved=True) # add block to meta db
2020-02-04 20:46:17 +00:00
spawn(
local_command,
f' /daemon-event/upload_event' ,
post=True,
is_json=True,
postData={' block' : blockHash}
)
2019-09-23 23:20:14 +00:00
blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database
else:
logger.warn(' POW failed for block %s.' % (blockHash,))
else:
if blacklist.inBlacklist(realHash):
logger.warn(' Block %s is blacklisted.' % (realHash,))
else:
logger.warn(' Metadata for block %s is invalid.' % (blockHash,))
blacklist.addToDB(blockHash)
else:
# if block didn' t meet expected hash
tempHash = onionrcrypto.hashers.sha3_hash(content) # lazy hack, TODO use var
try:
tempHash = tempHash.decode()
except AttributeError:
pass
# Punish peer for sharing invalid block (not always malicious, but is bad regardless)
onionrpeers.PeerProfiles(peerUsed).addScore(-50)
if tempHash != ' ed55e34cb828232d6c14da0479709bfa10a0923dca2b380496e6b2ed4f7a0253' :
# Dumb hack for 404 response from peer. Don' t log it if 404 since its likely not malicious or a critical error.
2020-02-04 20:46:17 +00:00
logger.warn(
' Block hash validation failed for ' +
blockHash + ' got ' + tempHash)
2019-09-23 23:20:14 +00:00
else:
removeFromQueue = False # Don' t remove from queue if 404
if removeFromQueue:
try:
del comm_inst.blockQueue[blockHash] # remove from block queue both if success or false
if count == LOG_SKIP_COUNT:
2020-02-04 20:46:17 +00:00
logger.info(' %s blocks remaining in queue' %
[len(comm_inst.blockQueue)], terminal=True)
2019-09-23 23:20:14 +00:00
count = 0
except KeyError:
pass
comm_inst.currentDownloading.remove(blockHash)
comm_inst.decrementThreadCount(' getBlocks' )< / code > < / pre >
< / details >
< / section >
< section >
< h2 class = "section-title" id = "header-submodules" > Sub-modules< / h2 >
< dl >
2020-02-04 20:46:17 +00:00
< dt > < code class = "name" > < a title = "src.communicatorutils.downloadblocks.shoulddownload" href = "shoulddownload.html" > src.communicatorutils.downloadblocks.shoulddownload< / a > < / code > < / dt >
2019-09-23 23:20:14 +00:00
< dd >
< section class = "desc" > < p > Onionr - Private P2P Communication …< / p > < / section >
< / dd >
< / dl >
< / section >
< section >
< / section >
< section >
< h2 class = "section-title" id = "header-functions" > Functions< / h2 >
< dl >
2020-02-04 20:46:17 +00:00
< dt id = "src.communicatorutils.downloadblocks.download_blocks_from_communicator" > < code class = "name flex" >
2019-09-23 23:20:14 +00:00
< span > def < span class = "ident" > download_blocks_from_communicator< / span > < / span > (< span > comm_inst)< / span >
< / code > < / dt >
< dd >
2020-02-04 20:46:17 +00:00
< section class = "desc" > < p > Use communicator instance to download blocks in the comms's queue< / p > < / section >
2019-09-23 23:20:14 +00:00
< details class = "source" >
2020-02-04 20:46:17 +00:00
< summary >
< span > Expand source code< / span >
< / summary >
< pre > < code class = "python" > def download_blocks_from_communicator(comm_inst: " OnionrCommunicatorDaemon" ):
" " " Use communicator instance to download blocks in the comms' s queue" " "
2019-09-23 23:20:14 +00:00
blacklist = onionrblacklist.OnionrBlackList()
storage_counter = storagecounter.StorageCounter()
LOG_SKIP_COUNT = 50 # for how many iterations we skip logging the counter
count: int = 0
metadata_validation_result: bool = False
# Iterate the block queue in the communicator
for blockHash in list(comm_inst.blockQueue):
count += 1
2020-02-04 20:46:17 +00:00
2019-09-23 23:20:14 +00:00
triedQueuePeers = [] # List of peers we' ve tried for a block
try:
blockPeers = list(comm_inst.blockQueue[blockHash])
except KeyError:
blockPeers = []
removeFromQueue = True
if not shoulddownload.should_download(comm_inst, blockHash):
continue
if comm_inst.shutdown or not comm_inst.isOnline or storage_counter.is_full():
# Exit loop if shutting down or offline, or disk allocation reached
break
# Do not download blocks being downloaded
if blockHash in comm_inst.currentDownloading:
continue
2020-02-04 20:46:17 +00:00
if len(comm_inst.onlinePeers) == 0:
break
2019-09-23 23:20:14 +00:00
comm_inst.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block
if len(blockPeers) == 0:
2020-02-04 20:46:17 +00:00
try:
peerUsed = onlinepeers.pick_online_peer(comm_inst)
except onionrexceptions.OnlinePeerNeeded:
continue
2019-09-23 23:20:14 +00:00
else:
blockPeers = onionrcrypto.cryptoutils.random_shuffle(blockPeers)
peerUsed = blockPeers.pop(0)
if not comm_inst.shutdown and peerUsed.strip() != ' ' :
logger.info(" Attempting to download %s from %s..." % (blockHash[:12], peerUsed))
content = peeraction.peer_action(comm_inst, peerUsed, ' getdata/' + blockHash, max_resp_size=3000000) # block content from random peer (includes metadata)
2020-02-04 20:46:17 +00:00
if content is not False and len(content) > 0:
2019-09-23 23:20:14 +00:00
try:
content = content.encode()
except AttributeError:
pass
realHash = onionrcrypto.hashers.sha3_hash(content)
try:
realHash = realHash.decode() # bytes on some versions for some reason
except AttributeError:
pass
if realHash == blockHash:
#content = content.decode() # decode here because sha3Hash needs bytes above
metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
metadata = metas[0]
try:
2020-02-04 20:46:17 +00:00
metadata_validation_result = \
validatemetadata.validate_metadata(metadata, metas[2])
2019-09-23 23:20:14 +00:00
except onionrexceptions.DataExists:
metadata_validation_result = False
if metadata_validation_result: # check if metadata is valid, and verify nonce
if onionrcrypto.cryptoutils.verify_POW(content): # check if POW is enough/correct
logger.info(' Attempting to save block %s...' % blockHash[:12])
try:
onionrstorage.set_data(content)
except onionrexceptions.DataExists:
logger.warn(' Data is already set for %s ' % (blockHash,))
except onionrexceptions.DiskAllocationReached:
logger.error(' Reached disk allocation allowance, cannot save block %s.' % (blockHash,))
removeFromQueue = False
else:
blockmetadb.add_to_block_DB(blockHash, dataSaved=True) # add block to meta db
2020-02-04 20:46:17 +00:00
spawn(
local_command,
f' /daemon-event/upload_event' ,
post=True,
is_json=True,
postData={' block' : blockHash}
)
2019-09-23 23:20:14 +00:00
blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database
else:
logger.warn(' POW failed for block %s.' % (blockHash,))
else:
if blacklist.inBlacklist(realHash):
logger.warn(' Block %s is blacklisted.' % (realHash,))
else:
logger.warn(' Metadata for block %s is invalid.' % (blockHash,))
blacklist.addToDB(blockHash)
else:
# if block didn' t meet expected hash
tempHash = onionrcrypto.hashers.sha3_hash(content) # lazy hack, TODO use var
try:
tempHash = tempHash.decode()
except AttributeError:
pass
# Punish peer for sharing invalid block (not always malicious, but is bad regardless)
onionrpeers.PeerProfiles(peerUsed).addScore(-50)
if tempHash != ' ed55e34cb828232d6c14da0479709bfa10a0923dca2b380496e6b2ed4f7a0253' :
# Dumb hack for 404 response from peer. Don' t log it if 404 since its likely not malicious or a critical error.
2020-02-04 20:46:17 +00:00
logger.warn(
' Block hash validation failed for ' +
blockHash + ' got ' + tempHash)
2019-09-23 23:20:14 +00:00
else:
removeFromQueue = False # Don' t remove from queue if 404
if removeFromQueue:
try:
del comm_inst.blockQueue[blockHash] # remove from block queue both if success or false
if count == LOG_SKIP_COUNT:
2020-02-04 20:46:17 +00:00
logger.info(' %s blocks remaining in queue' %
[len(comm_inst.blockQueue)], terminal=True)
2019-09-23 23:20:14 +00:00
count = 0
except KeyError:
pass
comm_inst.currentDownloading.remove(blockHash)
comm_inst.decrementThreadCount(' getBlocks' )< / code > < / pre >
< / details >
< / dd >
< / dl >
< / section >
< section >
< / section >
< / article >
< nav id = "sidebar" >
< h1 > Index< / h1 >
< div class = "toc" >
< ul > < / ul >
< / div >
< ul id = "index" >
< li > < h3 > Super-module< / h3 >
< ul >
2020-02-04 20:46:17 +00:00
< li > < code > < a title = "src.communicatorutils" href = "../index.html" > src.communicatorutils< / a > < / code > < / li >
2019-09-23 23:20:14 +00:00
< / ul >
< / li >
< li > < h3 > < a href = "#header-submodules" > Sub-modules< / a > < / h3 >
< ul >
2020-02-04 20:46:17 +00:00
< li > < code > < a title = "src.communicatorutils.downloadblocks.shoulddownload" href = "shoulddownload.html" > src.communicatorutils.downloadblocks.shoulddownload< / a > < / code > < / li >
2019-09-23 23:20:14 +00:00
< / ul >
< / li >
< li > < h3 > < a href = "#header-functions" > Functions< / a > < / h3 >
< ul class = "" >
2020-02-04 20:46:17 +00:00
< li > < code > < a title = "src.communicatorutils.downloadblocks.download_blocks_from_communicator" href = "#src.communicatorutils.downloadblocks.download_blocks_from_communicator" > download_blocks_from_communicator< / a > < / code > < / li >
2019-09-23 23:20:14 +00:00
< / ul >
< / li >
< / ul >
< / nav >
< / main >
< footer id = "footer" >
2020-02-04 20:46:17 +00:00
< p > Generated by < a href = "https://pdoc3.github.io/pdoc" > < cite > pdoc< / cite > 0.7.4< / a > .< / p >
2019-09-23 23:20:14 +00:00
< / footer >
< script src = "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js" > < / script >
< script > hljs . initHighlightingOnLoad ( ) < / script >
< / body >
< / html >
