2020-10-21 09:46:05 +00:00
|
|
|
"""Onionr - Private P2P Communication.
|
2019-05-09 05:27:15 +00:00
|
|
|
|
2020-10-21 09:46:05 +00:00
|
|
|
Download blocks using the communicator instance.
|
2019-12-22 19:42:10 +00:00
|
|
|
"""
|
|
|
|
from typing import TYPE_CHECKING
|
2020-10-21 09:46:05 +00:00
|
|
|
from secrets import SystemRandom
|
|
|
|
|
2019-12-22 19:42:10 +00:00
|
|
|
if TYPE_CHECKING:
|
|
|
|
from communicator import OnionrCommunicatorDaemon
|
2020-07-24 19:37:01 +00:00
|
|
|
from deadsimplekv import DeadSimpleKV
|
2020-01-04 12:13:10 +00:00
|
|
|
|
|
|
|
from gevent import spawn
|
|
|
|
|
2019-11-27 00:01:32 +00:00
|
|
|
import onionrexceptions
|
|
|
|
import logger
|
|
|
|
import onionrpeers
|
2020-07-24 08:24:41 +00:00
|
|
|
|
2019-11-27 00:01:32 +00:00
|
|
|
from communicator import peeraction
|
|
|
|
from communicator import onlinepeers
|
2020-11-02 01:31:11 +00:00
|
|
|
from onionrblocks import blockmetadata
|
2019-11-27 00:01:32 +00:00
|
|
|
from onionrutils import validatemetadata
|
|
|
|
from coredb import blockmetadb
|
2020-01-04 12:13:10 +00:00
|
|
|
from onionrutils.localcommand import local_command
|
2019-11-27 00:01:32 +00:00
|
|
|
import onionrcrypto
|
|
|
|
import onionrstorage
|
|
|
|
from onionrblocks import onionrblacklist
|
|
|
|
from onionrblocks import storagecounter
|
2022-01-06 20:48:22 +00:00
|
|
|
from onionrproofs import vdf
|
2019-11-27 00:01:32 +00:00
|
|
|
from . import shoulddownload
|
2019-12-22 19:42:10 +00:00
|
|
|
"""
|
2019-05-09 05:27:15 +00:00
|
|
|
This program is free software: you can redistribute it and/or modify
|
|
|
|
it under the terms of the GNU General Public License as published by
|
|
|
|
the Free Software Foundation, either version 3 of the License, or
|
|
|
|
(at your option) any later version.
|
|
|
|
|
|
|
|
This program is distributed in the hope that it will be useful,
|
|
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
GNU General Public License for more details.
|
|
|
|
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
|
|
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
2019-12-22 19:42:10 +00:00
|
|
|
"""
|
2019-11-27 00:01:32 +00:00
|
|
|
|
2020-07-24 08:24:41 +00:00
|
|
|
storage_counter = storagecounter.StorageCounter()
|
|
|
|
|
2019-11-27 00:01:32 +00:00
|
|
|
|
2020-11-15 18:26:25 +00:00
|
|
|
def download_blocks_from_communicator(shared_state: "TooMany"):
|
2019-12-22 19:42:10 +00:00
|
|
|
"""Use communicator instance to download blocks in the comms's queue"""
|
2019-07-18 17:40:48 +00:00
|
|
|
blacklist = onionrblacklist.OnionrBlackList()
|
2020-11-15 18:26:25 +00:00
|
|
|
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
|
2020-07-26 20:26:15 +00:00
|
|
|
LOG_SKIP_COUNT = 50 # for how many iterations we skip logging the counter
|
2019-09-17 01:16:06 +00:00
|
|
|
count: int = 0
|
|
|
|
metadata_validation_result: bool = False
|
2019-08-13 22:28:53 +00:00
|
|
|
# Iterate the block queue in the communicator
|
2020-07-24 19:37:01 +00:00
|
|
|
for blockHash in list(kv.get('blockQueue')):
|
2019-08-13 09:16:11 +00:00
|
|
|
count += 1
|
2019-12-20 07:24:38 +00:00
|
|
|
|
2019-05-08 03:28:06 +00:00
|
|
|
try:
|
2020-07-24 19:37:01 +00:00
|
|
|
blockPeers = list(kv.get('blockQueue')[blockHash])
|
2019-05-08 03:28:06 +00:00
|
|
|
except KeyError:
|
|
|
|
blockPeers = []
|
|
|
|
removeFromQueue = True
|
2019-07-04 02:28:16 +00:00
|
|
|
|
2020-11-15 18:26:25 +00:00
|
|
|
if not shoulddownload.should_download(shared_state, blockHash):
|
2019-07-04 02:28:16 +00:00
|
|
|
continue
|
|
|
|
|
2020-07-29 09:32:09 +00:00
|
|
|
if kv.get('shutdown') or not kv.get('isOnline') or \
|
2020-07-24 19:37:01 +00:00
|
|
|
storage_counter.is_full():
|
2019-07-04 02:28:16 +00:00
|
|
|
# Exit loop if shutting down or offline, or disk allocation reached
|
2019-05-08 03:28:06 +00:00
|
|
|
break
|
2019-07-09 06:25:20 +00:00
|
|
|
# Do not download blocks being downloaded
|
2020-07-26 20:26:15 +00:00
|
|
|
if blockHash in kv.get('currentDownloading'):
|
2019-05-08 03:28:06 +00:00
|
|
|
continue
|
2019-07-04 02:28:16 +00:00
|
|
|
|
2020-07-26 20:26:15 +00:00
|
|
|
if len(kv.get('onlinePeers')) == 0:
|
2019-12-20 07:24:38 +00:00
|
|
|
break
|
|
|
|
|
2020-07-26 20:26:15 +00:00
|
|
|
# So we can avoid concurrent downloading in other threads of same block
|
|
|
|
kv.get('currentDownloading').append(blockHash)
|
2019-05-08 03:28:06 +00:00
|
|
|
if len(blockPeers) == 0:
|
2019-12-20 07:24:38 +00:00
|
|
|
try:
|
2020-08-08 17:38:14 +00:00
|
|
|
peerUsed = onlinepeers.pick_online_peer(kv)
|
2019-12-20 07:24:38 +00:00
|
|
|
except onionrexceptions.OnlinePeerNeeded:
|
|
|
|
continue
|
2019-05-08 03:28:06 +00:00
|
|
|
else:
|
2020-10-21 09:46:05 +00:00
|
|
|
SystemRandom().shuffle(blockPeers)
|
2019-05-08 03:28:06 +00:00
|
|
|
peerUsed = blockPeers.pop(0)
|
|
|
|
|
2020-07-26 02:36:48 +00:00
|
|
|
if not kv.get('shutdown') and peerUsed.strip() != '':
|
2020-07-24 19:37:01 +00:00
|
|
|
logger.info(
|
|
|
|
f"Attempting to download %s from {peerUsed}..." % (blockHash[:12],))
|
|
|
|
content = peeraction.peer_action(
|
2020-11-15 18:26:25 +00:00
|
|
|
shared_state, peerUsed,
|
2020-07-24 19:37:01 +00:00
|
|
|
'getdata/' + blockHash,
|
|
|
|
max_resp_size=3000000) # block content from random peer
|
2019-09-11 21:50:09 +00:00
|
|
|
|
2020-01-04 12:13:10 +00:00
|
|
|
if content is not False and len(content) > 0:
|
2019-05-08 03:28:06 +00:00
|
|
|
try:
|
|
|
|
content = content.encode()
|
|
|
|
except AttributeError:
|
|
|
|
pass
|
|
|
|
|
2022-01-06 20:48:22 +00:00
|
|
|
if vdf.verify_vdf(blockHash, content):
|
2019-09-12 06:28:20 +00:00
|
|
|
#content = content.decode() # decode here because sha3Hash needs bytes above
|
2019-06-25 08:21:36 +00:00
|
|
|
metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
|
2019-05-08 03:28:06 +00:00
|
|
|
metadata = metas[0]
|
2019-09-17 01:16:06 +00:00
|
|
|
try:
|
2020-01-04 12:13:10 +00:00
|
|
|
metadata_validation_result = \
|
|
|
|
validatemetadata.validate_metadata(metadata, metas[2])
|
2020-10-10 23:23:22 +00:00
|
|
|
except onionrexceptions.PlaintextNotSupported:
|
|
|
|
logger.debug(f"Not saving {blockHash} due to plaintext not enabled")
|
|
|
|
removeFromQueue = True
|
2019-09-17 01:16:06 +00:00
|
|
|
except onionrexceptions.DataExists:
|
|
|
|
metadata_validation_result = False
|
|
|
|
if metadata_validation_result: # check if metadata is valid, and verify nonce
|
2022-01-06 20:48:22 +00:00
|
|
|
save_block(blockHash, data)
|
2019-05-08 03:28:06 +00:00
|
|
|
else:
|
2022-01-06 20:48:22 +00:00
|
|
|
if blacklist.inBlacklist(blockHash):
|
|
|
|
logger.warn(f'Block {blockHash} is blacklisted.')
|
2019-05-08 03:28:06 +00:00
|
|
|
else:
|
2019-06-29 18:18:31 +00:00
|
|
|
logger.warn('Metadata for block %s is invalid.' % (blockHash,))
|
2019-07-18 17:40:48 +00:00
|
|
|
blacklist.addToDB(blockHash)
|
2019-05-08 03:28:06 +00:00
|
|
|
else:
|
|
|
|
# if block didn't meet expected hash
|
2019-08-26 02:18:09 +00:00
|
|
|
tempHash = onionrcrypto.hashers.sha3_hash(content) # lazy hack, TODO use var
|
2019-05-08 03:28:06 +00:00
|
|
|
try:
|
|
|
|
tempHash = tempHash.decode()
|
|
|
|
except AttributeError:
|
|
|
|
pass
|
|
|
|
# Punish peer for sharing invalid block (not always malicious, but is bad regardless)
|
2019-07-18 17:40:48 +00:00
|
|
|
onionrpeers.PeerProfiles(peerUsed).addScore(-50)
|
2019-05-08 03:28:06 +00:00
|
|
|
if removeFromQueue:
|
|
|
|
try:
|
2020-07-24 19:37:01 +00:00
|
|
|
del kv.get('blockQueue')[blockHash] # remove from block queue both if success or false
|
2019-08-13 09:16:11 +00:00
|
|
|
if count == LOG_SKIP_COUNT:
|
2020-01-04 12:13:10 +00:00
|
|
|
logger.info('%s blocks remaining in queue' %
|
2020-07-24 19:37:01 +00:00
|
|
|
[len(kv.get('blockQueue'))], terminal=True)
|
2019-08-13 09:16:11 +00:00
|
|
|
count = 0
|
2019-05-08 03:28:06 +00:00
|
|
|
except KeyError:
|
|
|
|
pass
|
2020-07-26 20:26:15 +00:00
|
|
|
kv.get('currentDownloading').remove(blockHash)
|