Onionr/src/communicatorutils/downloadblocks/__init__.py

171 lines
7.5 KiB
Python
Raw Normal View History

2019-12-22 19:42:10 +00:00
"""
Onionr - Private P2P Communication
Download blocks using the communicator instance
2019-12-22 19:42:10 +00:00
"""
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from communicator import OnionrCommunicatorDaemon
from deadsimplekv import DeadSimpleKV
2020-01-04 12:13:10 +00:00
from gevent import spawn
import onionrexceptions
import logger
import onionrpeers
from communicator import peeraction
from communicator import onlinepeers
from onionrutils import blockmetadata
from onionrutils import validatemetadata
from coredb import blockmetadb
2020-01-04 12:13:10 +00:00
from onionrutils.localcommand import local_command
import onionrcrypto
import onionrstorage
from onionrblocks import onionrblacklist
from onionrblocks import storagecounter
from . import shoulddownload
2019-12-22 19:42:10 +00:00
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
2019-12-22 19:42:10 +00:00
"""
storage_counter = storagecounter.StorageCounter()
def download_blocks_from_communicator(comm_inst: "OnionrCommunicatorDaemon"):
2019-12-22 19:42:10 +00:00
"""Use communicator instance to download blocks in the comms's queue"""
2019-07-18 17:40:48 +00:00
blacklist = onionrblacklist.OnionrBlackList()
kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string("DeadSimpleKV")
LOG_SKIP_COUNT = 50 # for how many iterations we skip logging the counter
count: int = 0
metadata_validation_result: bool = False
# Iterate the block queue in the communicator
for blockHash in list(kv.get('blockQueue')):
count += 1
2019-05-08 03:28:06 +00:00
try:
blockPeers = list(kv.get('blockQueue')[blockHash])
2019-05-08 03:28:06 +00:00
except KeyError:
blockPeers = []
removeFromQueue = True
if not shoulddownload.should_download(comm_inst, blockHash):
continue
2020-07-29 09:32:09 +00:00
if kv.get('shutdown') or not kv.get('isOnline') or \
storage_counter.is_full():
# Exit loop if shutting down or offline, or disk allocation reached
2019-05-08 03:28:06 +00:00
break
2019-07-09 06:25:20 +00:00
# Do not download blocks being downloaded
if blockHash in kv.get('currentDownloading'):
2019-05-08 03:28:06 +00:00
continue
if len(kv.get('onlinePeers')) == 0:
break
# So we can avoid concurrent downloading in other threads of same block
kv.get('currentDownloading').append(blockHash)
2019-05-08 03:28:06 +00:00
if len(blockPeers) == 0:
try:
peerUsed = onlinepeers.pick_online_peer(kv)
except onionrexceptions.OnlinePeerNeeded:
continue
2019-05-08 03:28:06 +00:00
else:
2019-07-22 05:24:42 +00:00
blockPeers = onionrcrypto.cryptoutils.random_shuffle(blockPeers)
2019-05-08 03:28:06 +00:00
peerUsed = blockPeers.pop(0)
if not kv.get('shutdown') and peerUsed.strip() != '':
logger.info(
f"Attempting to download %s from {peerUsed}..." % (blockHash[:12],))
content = peeraction.peer_action(
comm_inst, peerUsed,
'getdata/' + blockHash,
max_resp_size=3000000) # block content from random peer
2020-01-04 12:13:10 +00:00
if content is not False and len(content) > 0:
2019-05-08 03:28:06 +00:00
try:
content = content.encode()
except AttributeError:
pass
2019-07-22 05:24:42 +00:00
realHash = onionrcrypto.hashers.sha3_hash(content)
2019-05-08 03:28:06 +00:00
try:
realHash = realHash.decode() # bytes on some versions for some reason
except AttributeError:
pass
if realHash == blockHash:
2019-09-12 06:28:20 +00:00
#content = content.decode() # decode here because sha3Hash needs bytes above
metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
2019-05-08 03:28:06 +00:00
metadata = metas[0]
try:
2020-01-04 12:13:10 +00:00
metadata_validation_result = \
validatemetadata.validate_metadata(metadata, metas[2])
except onionrexceptions.DataExists:
metadata_validation_result = False
if metadata_validation_result: # check if metadata is valid, and verify nonce
2019-07-22 05:24:42 +00:00
if onionrcrypto.cryptoutils.verify_POW(content): # check if POW is enough/correct
2019-05-08 03:28:06 +00:00
logger.info('Attempting to save block %s...' % blockHash[:12])
try:
2019-07-24 17:22:19 +00:00
onionrstorage.set_data(content)
2019-06-29 18:18:31 +00:00
except onionrexceptions.DataExists:
logger.warn('Data is already set for %s ' % (blockHash,))
2019-05-08 03:28:06 +00:00
except onionrexceptions.DiskAllocationReached:
2019-06-29 18:18:31 +00:00
logger.error('Reached disk allocation allowance, cannot save block %s.' % (blockHash,))
2019-05-08 03:28:06 +00:00
removeFromQueue = False
else:
blockmetadb.add_to_block_DB(blockHash, dataSaved=True) # add block to meta db
2020-02-26 02:30:04 +00:00
blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database
2020-01-04 12:13:10 +00:00
spawn(
local_command,
f'/daemon-event/upload_event',
post=True,
is_json=True,
2020-07-17 18:49:18 +00:00
post_data={'block': blockHash}
2020-01-04 12:13:10 +00:00
)
2019-05-08 03:28:06 +00:00
else:
2019-06-29 18:18:31 +00:00
logger.warn('POW failed for block %s.' % (blockHash,))
2019-05-08 03:28:06 +00:00
else:
2019-07-18 17:40:48 +00:00
if blacklist.inBlacklist(realHash):
2019-05-08 03:28:06 +00:00
logger.warn('Block %s is blacklisted.' % (realHash,))
else:
2019-06-29 18:18:31 +00:00
logger.warn('Metadata for block %s is invalid.' % (blockHash,))
2019-07-18 17:40:48 +00:00
blacklist.addToDB(blockHash)
2019-05-08 03:28:06 +00:00
else:
# if block didn't meet expected hash
2019-08-26 02:18:09 +00:00
tempHash = onionrcrypto.hashers.sha3_hash(content) # lazy hack, TODO use var
2019-05-08 03:28:06 +00:00
try:
tempHash = tempHash.decode()
except AttributeError:
pass
# Punish peer for sharing invalid block (not always malicious, but is bad regardless)
2019-07-18 17:40:48 +00:00
onionrpeers.PeerProfiles(peerUsed).addScore(-50)
2019-05-08 03:28:06 +00:00
if tempHash != 'ed55e34cb828232d6c14da0479709bfa10a0923dca2b380496e6b2ed4f7a0253':
# Dumb hack for 404 response from peer. Don't log it if 404 since its likely not malicious or a critical error.
2020-01-04 12:13:10 +00:00
logger.warn(
'Block hash validation failed for ' +
blockHash + ' got ' + tempHash)
2019-05-08 03:28:06 +00:00
else:
removeFromQueue = False # Don't remove from queue if 404
if removeFromQueue:
try:
del kv.get('blockQueue')[blockHash] # remove from block queue both if success or false
if count == LOG_SKIP_COUNT:
2020-01-04 12:13:10 +00:00
logger.info('%s blocks remaining in queue' %
[len(kv.get('blockQueue'))], terminal=True)
count = 0
2019-05-08 03:28:06 +00:00
except KeyError:
pass
kv.get('currentDownloading').remove(blockHash)
comm_inst.decrementThreadCount('getBlocks')