Onionr/src/communicatorutils/downloadblocks/__init__.py

174 lines
7.6 KiB
Python
Raw Normal View History

"""Onionr - Private P2P Communication.
Download blocks using the communicator instance.
2019-12-22 19:42:10 +00:00
"""
from typing import TYPE_CHECKING
from secrets import SystemRandom
2019-12-22 19:42:10 +00:00
if TYPE_CHECKING:
from communicator import OnionrCommunicatorDaemon
from deadsimplekv import DeadSimpleKV
2020-01-04 12:13:10 +00:00
from gevent import spawn
import onionrexceptions
import logger
import onionrpeers
from communicator import peeraction
from communicator import onlinepeers
2020-11-02 01:31:11 +00:00
from onionrblocks import blockmetadata
from onionrutils import validatemetadata
from coredb import blockmetadb
2020-01-04 12:13:10 +00:00
from onionrutils.localcommand import local_command
import onionrcrypto
import onionrstorage
from onionrblocks import onionrblacklist
from onionrblocks import storagecounter
from . import shoulddownload
2019-12-22 19:42:10 +00:00
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
2019-12-22 19:42:10 +00:00
"""
storage_counter = storagecounter.StorageCounter()
2020-11-15 18:26:25 +00:00
def download_blocks_from_communicator(shared_state: "TooMany"):
2019-12-22 19:42:10 +00:00
"""Use communicator instance to download blocks in the comms's queue"""
2019-07-18 17:40:48 +00:00
blacklist = onionrblacklist.OnionrBlackList()
2020-11-15 18:26:25 +00:00
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
LOG_SKIP_COUNT = 50 # for how many iterations we skip logging the counter
count: int = 0
metadata_validation_result: bool = False
# Iterate the block queue in the communicator
for blockHash in list(kv.get('blockQueue')):
count += 1
2019-05-08 03:28:06 +00:00
try:
blockPeers = list(kv.get('blockQueue')[blockHash])
2019-05-08 03:28:06 +00:00
except KeyError:
blockPeers = []
removeFromQueue = True
2020-11-15 18:26:25 +00:00
if not shoulddownload.should_download(shared_state, blockHash):
continue
2020-07-29 09:32:09 +00:00
if kv.get('shutdown') or not kv.get('isOnline') or \
storage_counter.is_full():
# Exit loop if shutting down or offline, or disk allocation reached
2019-05-08 03:28:06 +00:00
break
2019-07-09 06:25:20 +00:00
# Do not download blocks being downloaded
if blockHash in kv.get('currentDownloading'):
2019-05-08 03:28:06 +00:00
continue
if len(kv.get('onlinePeers')) == 0:
break
# So we can avoid concurrent downloading in other threads of same block
kv.get('currentDownloading').append(blockHash)
2019-05-08 03:28:06 +00:00
if len(blockPeers) == 0:
try:
peerUsed = onlinepeers.pick_online_peer(kv)
except onionrexceptions.OnlinePeerNeeded:
continue
2019-05-08 03:28:06 +00:00
else:
SystemRandom().shuffle(blockPeers)
2019-05-08 03:28:06 +00:00
peerUsed = blockPeers.pop(0)
if not kv.get('shutdown') and peerUsed.strip() != '':
logger.info(
f"Attempting to download %s from {peerUsed}..." % (blockHash[:12],))
content = peeraction.peer_action(
2020-11-15 18:26:25 +00:00
shared_state, peerUsed,
'getdata/' + blockHash,
max_resp_size=3000000) # block content from random peer
2020-01-04 12:13:10 +00:00
if content is not False and len(content) > 0:
2019-05-08 03:28:06 +00:00
try:
content = content.encode()
except AttributeError:
pass
2019-07-22 05:24:42 +00:00
realHash = onionrcrypto.hashers.sha3_hash(content)
2019-05-08 03:28:06 +00:00
try:
realHash = realHash.decode() # bytes on some versions for some reason
except AttributeError:
pass
if realHash == blockHash:
2019-09-12 06:28:20 +00:00
#content = content.decode() # decode here because sha3Hash needs bytes above
metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
2019-05-08 03:28:06 +00:00
metadata = metas[0]
try:
2020-01-04 12:13:10 +00:00
metadata_validation_result = \
validatemetadata.validate_metadata(metadata, metas[2])
except onionrexceptions.PlaintextNotSupported:
logger.debug(f"Not saving {blockHash} due to plaintext not enabled")
removeFromQueue = True
except onionrexceptions.DataExists:
metadata_validation_result = False
if metadata_validation_result: # check if metadata is valid, and verify nonce
2019-07-22 05:24:42 +00:00
if onionrcrypto.cryptoutils.verify_POW(content): # check if POW is enough/correct
2019-05-08 03:28:06 +00:00
logger.info('Attempting to save block %s...' % blockHash[:12])
try:
2019-07-24 17:22:19 +00:00
onionrstorage.set_data(content)
2019-06-29 18:18:31 +00:00
except onionrexceptions.DataExists:
logger.warn('Data is already set for %s ' % (blockHash,))
2019-05-08 03:28:06 +00:00
except onionrexceptions.DiskAllocationReached:
2019-06-29 18:18:31 +00:00
logger.error('Reached disk allocation allowance, cannot save block %s.' % (blockHash,))
2019-05-08 03:28:06 +00:00
removeFromQueue = False
else:
blockmetadb.add_to_block_DB(blockHash, dataSaved=True) # add block to meta db
2020-02-26 02:30:04 +00:00
blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database
2020-01-04 12:13:10 +00:00
spawn(
local_command,
f'/daemon-event/upload_event',
post=True,
is_json=True,
2020-07-17 18:49:18 +00:00
post_data={'block': blockHash}
2020-01-04 12:13:10 +00:00
)
2019-05-08 03:28:06 +00:00
else:
2019-06-29 18:18:31 +00:00
logger.warn('POW failed for block %s.' % (blockHash,))
2019-05-08 03:28:06 +00:00
else:
2019-07-18 17:40:48 +00:00
if blacklist.inBlacklist(realHash):
2019-05-08 03:28:06 +00:00
logger.warn('Block %s is blacklisted.' % (realHash,))
else:
2019-06-29 18:18:31 +00:00
logger.warn('Metadata for block %s is invalid.' % (blockHash,))
2019-07-18 17:40:48 +00:00
blacklist.addToDB(blockHash)
2019-05-08 03:28:06 +00:00
else:
# if block didn't meet expected hash
2019-08-26 02:18:09 +00:00
tempHash = onionrcrypto.hashers.sha3_hash(content) # lazy hack, TODO use var
2019-05-08 03:28:06 +00:00
try:
tempHash = tempHash.decode()
except AttributeError:
pass
# Punish peer for sharing invalid block (not always malicious, but is bad regardless)
2019-07-18 17:40:48 +00:00
onionrpeers.PeerProfiles(peerUsed).addScore(-50)
2019-05-08 03:28:06 +00:00
if tempHash != 'ed55e34cb828232d6c14da0479709bfa10a0923dca2b380496e6b2ed4f7a0253':
# Dumb hack for 404 response from peer. Don't log it if 404 since its likely not malicious or a critical error.
2020-01-04 12:13:10 +00:00
logger.warn(
'Block hash validation failed for ' +
blockHash + ' got ' + tempHash)
2019-05-08 03:28:06 +00:00
else:
removeFromQueue = False # Don't remove from queue if 404
if removeFromQueue:
try:
del kv.get('blockQueue')[blockHash] # remove from block queue both if success or false
if count == LOG_SKIP_COUNT:
2020-01-04 12:13:10 +00:00
logger.info('%s blocks remaining in queue' %
[len(kv.get('blockQueue'))], terminal=True)
count = 0
2019-05-08 03:28:06 +00:00
except KeyError:
pass
kv.get('currentDownloading').remove(blockHash)