linting refactoring communicator(utils) and reduced TOCTOU issus with online peer picking

This commit is contained in:
Kevin Froman 2019-12-20 01:24:38 -06:00
parent 9fbee668aa
commit e5f3866f9e
8 changed files with 88 additions and 45 deletions

View File

@ -1,9 +1,13 @@
''' """Onionr - Private P2P Communication.
Onionr - Private P2P Communication
clear offline peer in a communicator instance clear offline peer in a communicator instance
''' """
''' from typing import TYPE_CHECKING
import logger
if TYPE_CHECKING:
from communicator import OnionrCommunicatorDaemon
"""
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or the Free Software Foundation, either version 3 of the License, or
@ -16,14 +20,16 @@
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' """
import logger
def clear_offline_peer(comm_inst):
'''Removes the longest offline peer to retry later''' def clear_offline_peer(comm_inst: 'OnionrCommunicatorDaemon'):
"""Remove the longest offline peer to retry later."""
try: try:
removed = comm_inst.offlinePeers.pop(0) removed = comm_inst.offlinePeers.pop(0)
except IndexError: except IndexError:
pass pass
else: else:
logger.debug('Removed ' + removed + ' from offline list, will try them again.') logger.debug('Removed ' + removed +
' from offline list, will try them again.')
comm_inst.decrementThreadCount('clear_offline_peer') comm_inst.decrementThreadCount('clear_offline_peer')

View File

@ -1,11 +1,14 @@
""" """Onionr - Private P2P Communication.
Onionr - Private P2P Communication
get online peers in a communicator instance get online peers in a communicator instance
""" """
import time import time
from typing import TYPE_CHECKING
from etc import humanreadabletime from etc import humanreadabletime
import logger import logger
if TYPE_CHECKING:
from communicator import OnionrCommunicatorDaemon
""" """
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
@ -22,25 +25,25 @@ import logger
""" """
def get_online_peers(comm_inst): def get_online_peers(comm_inst: 'OnionrCommunicatorDaemon'):
""" """Manage the comm_inst.onlinePeers attribute list.
Manages the comm_inst.onlinePeers attribute list,
]connects to more peers if we have none connected Connect to more peers if we have none connected
""" """
config = comm_inst.config config = comm_inst.config
if config.get('general.offline_mode', False): if config.get('general.offline_mode', False):
comm_inst.decrementThreadCount('get_online_peers') comm_inst.decrementThreadCount('get_online_peers')
return return
logger.debug('Refreshing peer pool...') logger.debug('Refreshing peer pool...')
maxPeers = int(config.get('peers.max_connect', 10)) max_peers = int(config.get('peers.max_connect', 10))
needed = maxPeers - len(comm_inst.onlinePeers) needed = max_peers - len(comm_inst.onlinePeers)
last_seen = 'never' last_seen = 'never'
if not isinstance(comm_inst.lastNodeSeen, type(None)): if not isinstance(comm_inst.lastNodeSeen, type(None)):
last_seen = humanreadabletime.human_readable_time( last_seen = humanreadabletime.human_readable_time(
comm_inst.lastNodeSeen) comm_inst.lastNodeSeen)
for i in range(needed): for _ in range(needed):
if len(comm_inst.onlinePeers) == 0: if len(comm_inst.onlinePeers) == 0:
comm_inst.connectNewPeer(useBootstrap=True) comm_inst.connectNewPeer(useBootstrap=True)
else: else:
@ -50,8 +53,7 @@ def get_online_peers(comm_inst):
break break
else: else:
if len(comm_inst.onlinePeers) == 0: if len(comm_inst.onlinePeers) == 0:
logger.debug logger.debug('Couldn\'t connect to any peers.' +
('Couldn\'t connect to any peers.' +
f' Last node seen {last_seen} ago.') f' Last node seen {last_seen} ago.')
else: else:
comm_inst.lastNodeSeen = time.time() comm_inst.lastNodeSeen = time.time()

View File

@ -1,9 +1,12 @@
''' """
Onionr - Private P2P Communication Onionr - Private P2P Communication.
pick online peers in a communicator instance pick online peers in a communicator instance
''' """
''' import secrets
import onionrexceptions
"""
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or the Free Software Foundation, either version 3 of the License, or
@ -16,17 +19,22 @@
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' """
import secrets
def pick_online_peer(comm_inst): def pick_online_peer(comm_inst):
'''randomly picks peer from pool without bias (using secrets module)''' """Randomly picks peer from pool without bias (using secrets module)."""
ret_data = '' ret_data = ''
while True:
peer_length = len(comm_inst.onlinePeers) peer_length = len(comm_inst.onlinePeers)
if peer_length <= 0: if peer_length <= 0:
break raise onionrexceptions.OnlinePeerNeeded
while True:
peer_length = len(comm_inst.onlinePeers)
try: try:
# get a random online peer, securely. May get stuck in loop if network is lost or if all peers in pool magically disconnect at once # Get a random online peer, securely.
# May get stuck in loop if network is lost or if all peers in pool magically disconnect at once
ret_data = comm_inst.onlinePeers[secrets.randbelow(peer_length)] ret_data = comm_inst.onlinePeers[secrets.randbelow(peer_length)]
except IndexError: except IndexError:
pass pass

View File

@ -25,6 +25,8 @@ from utils import gettransports
from netcontroller import NetController from netcontroller import NetController
from communicator import onlinepeers from communicator import onlinepeers
from coredb import keydb from coredb import keydb
import onionrexceptions
def announce_node(daemon): def announce_node(daemon):
'''Announce our node to our peers''' '''Announce our node to our peers'''
ret_data = False ret_data = False
@ -41,12 +43,17 @@ def announce_node(daemon):
peer = i peer = i
break break
else: else:
try:
peer = onlinepeers.pick_online_peer(daemon) peer = onlinepeers.pick_online_peer(daemon)
except onionrexceptions.OnlinePeerNeeded:
peer = ""
for x in range(1): for _ in range(1):
try: try:
ourID = gettransports.get()[0] ourID = gettransports.get()[0]
except IndexError: if not peer:
raise onionrexceptions.OnlinePeerNeeded
except (IndexError, onionrexceptions.OnlinePeerNeeded):
break break
url = 'http://' + peer + '/announce' url = 'http://' + peer + '/announce'

View File

@ -43,8 +43,7 @@ def download_blocks_from_communicator(comm_inst: "OnionrCommunicatorDaemon"):
# Iterate the block queue in the communicator # Iterate the block queue in the communicator
for blockHash in list(comm_inst.blockQueue): for blockHash in list(comm_inst.blockQueue):
count += 1 count += 1
if len(comm_inst.onlinePeers) == 0:
break
triedQueuePeers = [] # List of peers we've tried for a block triedQueuePeers = [] # List of peers we've tried for a block
try: try:
blockPeers = list(comm_inst.blockQueue[blockHash]) blockPeers = list(comm_inst.blockQueue[blockHash])
@ -62,9 +61,15 @@ def download_blocks_from_communicator(comm_inst: "OnionrCommunicatorDaemon"):
if blockHash in comm_inst.currentDownloading: if blockHash in comm_inst.currentDownloading:
continue continue
if len(comm_inst.onlinePeers) == 0:
break
comm_inst.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block comm_inst.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block
if len(blockPeers) == 0: if len(blockPeers) == 0:
try:
peerUsed = onlinepeers.pick_online_peer(comm_inst) peerUsed = onlinepeers.pick_online_peer(comm_inst)
except onionrexceptions.OnlinePeerNeeded:
continue
else: else:
blockPeers = onionrcrypto.cryptoutils.random_shuffle(blockPeers) blockPeers = onionrcrypto.cryptoutils.random_shuffle(blockPeers)
peerUsed = blockPeers.pop(0) peerUsed = blockPeers.pop(0)

View File

@ -21,6 +21,7 @@ import logger
from onionrutils import stringvalidators from onionrutils import stringvalidators
from communicator import peeraction, onlinepeers from communicator import peeraction, onlinepeers
from utils import gettransports from utils import gettransports
import onionrexceptions
def lookup_new_peer_transports_with_communicator(comm_inst): def lookup_new_peer_transports_with_communicator(comm_inst):
logger.info('Looking up new addresses...') logger.info('Looking up new addresses...')
tryAmount = 1 tryAmount = 1
@ -32,8 +33,11 @@ def lookup_new_peer_transports_with_communicator(comm_inst):
if len(newPeers) > 10000: if len(newPeers) > 10000:
# Don't get new peers if we have too many queued up # Don't get new peers if we have too many queued up
break break
try:
peer = onlinepeers.pick_online_peer(comm_inst) peer = onlinepeers.pick_online_peer(comm_inst)
newAdders = peeraction.peer_action(comm_inst, peer, action='pex') newAdders = peeraction.peer_action(comm_inst, peer, action='pex')
except onionrexceptions.OnlinePeerNeeded:
continue
try: try:
newPeers = newAdders.split(',') newPeers = newAdders.split(',')
except AttributeError: except AttributeError:

View File

@ -17,12 +17,15 @@
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
from gevent import time
import logger, onionrproofs import logger, onionrproofs
from onionrutils import stringvalidators, epoch from onionrutils import stringvalidators, epoch
from communicator import peeraction, onlinepeers from communicator import peeraction, onlinepeers
from coredb import blockmetadb from coredb import blockmetadb
from utils import reconstructhash from utils import reconstructhash
from onionrblocks import onionrblacklist from onionrblocks import onionrblacklist
import onionrexceptions
blacklist = onionrblacklist.OnionrBlackList() blacklist = onionrblacklist.OnionrBlackList()
def lookup_blocks_from_communicator(comm_inst): def lookup_blocks_from_communicator(comm_inst):
logger.info('Looking up new blocks') logger.info('Looking up new blocks')
@ -43,7 +46,12 @@ def lookup_blocks_from_communicator(comm_inst):
if comm_inst.storage_counter.is_full(): if comm_inst.storage_counter.is_full():
logger.debug('Not looking up new blocks due to maximum amount of allowed disk space used') logger.debug('Not looking up new blocks due to maximum amount of allowed disk space used')
break break
peer = onlinepeers.pick_online_peer(comm_inst) # select random online peer try:
# select random online peer
peer = onlinepeers.pick_online_peer(comm_inst)
except onionrexceptions.OnlinePeerNeeded:
time.sleep(1)
continue
# if we've already tried all the online peers this time around, stop # if we've already tried all the online peers this time around, stop
if peer in triedPeers: if peer in triedPeers:
if len(comm_inst.onlinePeers) == len(triedPeers): if len(comm_inst.onlinePeers) == len(triedPeers):

View File

@ -45,8 +45,11 @@ def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
comm_inst.decrementThreadCount(TIMER_NAME) comm_inst.decrementThreadCount(TIMER_NAME)
return return
session = session_manager.add_session(bl) session = session_manager.add_session(bl)
for i in range(min(len(comm_inst.onlinePeers), 6)): for _ in range(min(len(comm_inst.onlinePeers), 6)):
try:
peer = onlinepeers.pick_online_peer(comm_inst) peer = onlinepeers.pick_online_peer(comm_inst)
except onionrexceptions.OnlinePeerNeeded:
continue
try: try:
session.peer_exists[peer] session.peer_exists[peer]
continue continue