kev/Onionr
1
0
Fork 0
Code Issues 24 Pull Requests Projects Releases 2 Wiki Activity

formatting and comment improvements, cleanup

Browse Source
This commit is contained in:
Kevin 2020-07-07 08:37:23 -05:00
parent 714b3a3c33
commit 782c980b69
13 changed files with 207 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/communicatorutils/announcenode.py
View File

@ -1,14 +1,11 @@
"""
Onionr - Private P2P Communication.
Onionr - Private P2P Communication.
Use a communicator instance to announce
our transport address to connected nodes
"""
import base64
import onionrproofs
import logger
from etc import onionrvalues
from onionrutils import basicrequests, bytesconverter
from onionrutils import basicrequests
from utils import gettransports
from netcontroller import NetController
from communicator import onlinepeers
@ -33,7 +30,6 @@ import onionrexceptions
def announce_node(daemon):
"""Announce our node to our peers."""
ret_data = False
announce_fail = False
# Do not let announceCache get too large
if len(daemon.announceCache) >= 10000:

32
src/communicatorutils/connectnewpeers.py
View File

@ -1,9 +1,14 @@
"""Onionr - Private P2P Communication.
Connect a new peer to our communicator instance. Does so randomly if no peer is specified
Connect a new peer to our communicator instance.
Does so randomly if no peer is specified
"""
import time, sys, secrets
import onionrexceptions, logger, onionrpeers
import time
import secrets
import onionrexceptions
import logger
import onionrpeers
from utils import networkmerger, gettransports
from onionrutils import stringvalidators, epoch
from communicator import peeraction, bootstrappeers
@ -33,14 +38,18 @@ def connect_new_peer_to_communicator(comm_inst, peer='', useBootstrap=False):
if stringvalidators.validate_transport(peer):
peerList = [peer]
else:
raise onionrexceptions.InvalidAddress('Will not attempt connection test to invalid address')
raise onionrexceptions.InvalidAddress(
'Will not attempt connection test to invalid address')
else:
peerList = keydb.listkeys.list_adders()
mainPeerList = keydb.listkeys.list_adders()
peerList = onionrpeers.get_score_sorted_peer_list()
# If we don't have enough peers connected or random chance, select new peers to try
"""
If we don't have enough peers connected or random chance,
select new peers to try
"""
if len(peerList) < 8 or secrets.randbelow(4) == 3:
tryingNew = []
for x in comm_inst.newPeers:
@ -61,8 +70,12 @@ def connect_new_peer_to_communicator(comm_inst, peer='', useBootstrap=False):
# Don't connect to our own address
if address in transports:
continue
# Don't connect to invalid address or if its already been tried/connected, or if its cooled down
if len(address) == 0 or address in tried or address in comm_inst.onlinePeers or address in comm_inst.cooldownPeer:
"""Don't connect to invalid address or
if its already been tried/connected, or if its cooled down
"""
if len(address) == 0 or address in tried \
or address in comm_inst.onlinePeers \
or address in comm_inst.cooldownPeer:
continue
if comm_inst.shutdown:
return
@ -71,7 +84,7 @@ def connect_new_peer_to_communicator(comm_inst, peer='', useBootstrap=False):
if ret == 'pong!':
time.sleep(0.1)
if address not in mainPeerList:
# Add a peer to our list if it isn't already since it successfully connected
# Add a peer to our list if it isn't already since it connected
networkmerger.mergeAdders(address)
if address not in comm_inst.onlinePeers:
logger.info('Connected to ' + address, terminal=True)
@ -84,7 +97,8 @@ def connect_new_peer_to_communicator(comm_inst, peer='', useBootstrap=False):
if profile.address == address:
break
else:
comm_inst.peerProfiles.append(onionrpeers.PeerProfiles(address))
comm_inst.peerProfiles.append(
onionrpeers.PeerProfiles(address))
break
else:
# Mark a peer as tried if they failed to respond to ping

39
src/communicatorutils/cooldownpeer.py
View File

@ -1,9 +1,10 @@
'''
Onionr - Private P2P Communication
"""Onionr - Private P2P Communication.
Select a random online peer in a communicator instance and have them "cool down"
'''
'''
Select random online peer in a communicator instance and have them "cool down"
"""
from onionrutils import epoch
from communicator import onlinepeers
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@ -16,39 +17,39 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
from onionrutils import epoch
from communicator import onlinepeers
"""
def cooldown_peer(comm_inst):
'''Randomly add an online peer to cooldown, so we can connect a new one'''
"""Randomly add an online peer to cooldown, so we can connect a new one."""
config = comm_inst.config
onlinePeerAmount = len(comm_inst.onlinePeers)
online_peer_amount = len(comm_inst.onlinePeers)
minTime = 300
cooldownTime = 600
toCool = ''
cooldown_time = 600
to_cool = ''
tempConnectTimes = dict(comm_inst.connectTimes)
# Remove peers from cooldown that have been there long enough
tempCooldown = dict(comm_inst.cooldownPeer)
for peer in tempCooldown:
if (epoch.get_epoch() - tempCooldown[peer]) >= cooldownTime:
if (epoch.get_epoch() - tempCooldown[peer]) >= cooldown_time:
del comm_inst.cooldownPeer[peer]
# Cool down a peer, if we have max connections alive for long enough
if onlinePeerAmount >= config.get('peers.max_connect', 10, save = True):
if online_peer_amount >= config.get('peers.max_connect', 10, save=True):
finding = True
while finding:
try:
toCool = min(tempConnectTimes, key=tempConnectTimes.get)
if (epoch.get_epoch() - tempConnectTimes[toCool]) < minTime:
del tempConnectTimes[toCool]
to_cool = min(tempConnectTimes, key=tempConnectTimes.get)
if (epoch.get_epoch() - tempConnectTimes[to_cool]) < minTime:
del tempConnectTimes[to_cool]
else:
finding = False
except ValueError:
break
else:
onlinepeers.remove_online_peer(comm_inst, toCool)
comm_inst.cooldownPeer[toCool] = epoch.get_epoch()
onlinepeers.remove_online_peer(comm_inst, to_cool)
comm_inst.cooldownPeer[to_cool] = epoch.get_epoch()
comm_inst.decrementThreadCount('cooldown_peer')

26
src/communicatorutils/deniableinserts.py
View File

@ -1,9 +1,12 @@
'''
Onionr - Private P2P Communication
"""Onionr - Private P2P Communication.
Use the communicator to insert fake mail messages
'''
'''
Use the communicator to insert fake mail messages
"""
import secrets
from etc import onionrvalues
import onionrblocks
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@ -16,17 +19,18 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import secrets
from etc import onionrvalues
import onionrblocks
"""
def insert_deniable_block(comm_inst):
'''Insert a fake block in order to make it more difficult to track real blocks'''
"""Insert a fake block to make it more difficult to track real blocks."""
fakePeer = ''
chance = 10
if secrets.randbelow(chance) == (chance - 1):
# This assumes on the libsodium primitives to have key-privacy
fakePeer = onionrvalues.DENIABLE_PEER_ADDRESS
data = secrets.token_hex(secrets.randbelow(5120) + 1)
onionrblocks.insert(data, header='pm', encryptType='asym', asymPeer=fakePeer, disableForward=True, meta={'subject': 'foo'})
onionrblocks.insert(data, header='pm', encryptType='asym',
asymPeer=fakePeer, disableForward=True,
meta={'subject': 'foo'})
comm_inst.decrementThreadCount('insert_deniable_block')

4
src/communicatorutils/housekeeping.py
View File

@ -4,6 +4,7 @@ Cleanup old Onionr blocks and forward secrecy keys using the communicator.
Ran from a communicator timer usually
"""
import sqlite3
import logger
from onionrusers import onionrusers
from onionrutils import epoch
@ -67,7 +68,8 @@ def clean_keys(comm_inst):
time = epoch.get_epoch()
deleteKeys = []
for entry in c.execute("SELECT * FROM forwardKeys WHERE expire <= ?", (time,)):
for entry in c.execute(
"SELECT * FROM forwardKeys WHERE expire <= ?", (time,)):
logger.debug('Forward key: %s' % entry[1])
deleteKeys.append(entry[1])

30
src/communicatorutils/lookupadders.py
View File

@ -1,9 +1,14 @@
'''
Onionr - Private P2P Communication
"""
Onionr - Private P2P Communication.
Lookup new peer transport addresses using the communicator
'''
'''
Lookup new peer transport addresses using the communicator
"""
import logger
from onionrutils import stringvalidators
from communicator import peeraction, onlinepeers
from utils import gettransports
import onionrexceptions
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@ -16,12 +21,9 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import logger
from onionrutils import stringvalidators
from communicator import peeraction, onlinepeers
from utils import gettransports
import onionrexceptions
"""
def lookup_new_peer_transports_with_communicator(comm_inst):
logger.info('Looking up new addresses...')
tryAmount = 1
@ -47,7 +49,8 @@ def lookup_new_peer_transports_with_communicator(comm_inst):
invalid = []
for x in newPeers:
x = x.strip()
if not stringvalidators.validate_transport(x) or x in comm_inst.newPeers or x in transports:
if not stringvalidators.validate_transport(x) \
or x in comm_inst.newPeers or x in transports:
# avoid adding if its our address
invalid.append(x)
for x in invalid:
@ -56,4 +59,5 @@ def lookup_new_peer_transports_with_communicator(comm_inst):
except ValueError:
pass
comm_inst.newPeers.extend(newPeers)
comm_inst.decrementThreadCount('lookup_new_peer_transports_with_communicator')
comm_inst.decrementThreadCount(
'lookup_new_peer_transports_with_communicator')

16
src/communicatorutils/netcheck.py
View File

@ -1,9 +1,9 @@
"""
Onionr - Private P2P Communication
Onionr - Private P2P Communication.
Determine if our node is able to use Tor based
on the status of a communicator instance
and the result of pinging onion http servers
Determine if our node is able to use Tor based
on the status of a communicator instance
and the result of pinging onion http servers
"""
import logger
from utils import netutils
@ -26,8 +26,10 @@ from . import restarttor
def net_check(comm_inst):
"""Check if we are connected to the internet
or not when we can't connect to any peers"""
"""Check if we are connected to the internet.
or not when we can't connect to any peers
"""
# for detecting if we have received incoming connections recently
rec = False
if len(comm_inst.onlinePeers) == 0:
@ -43,7 +45,7 @@ def net_check(comm_inst):
if not comm_inst.config.get('general.offline_mode', False):
logger.warn('Network check failed, are you connected to ' +
'the Internet, and is Tor working? ' +
'This is usually temporary, but bugs and censorship can cause this to persist, in which case you should report it to beardog [at] mailbox.org',
'This is usually temporary, but bugs and censorship can cause this to persist, in which case you should report it to beardog [at] mailbox.org', # noqa
terminal=True)
restarttor.restart(comm_inst)
comm_inst.offlinePeers = []

68
src/communicatorutils/onionrcommunicatortimers.py
View File

@ -1,10 +1,20 @@
'''
Onionr - Private P2P Communication
"""
Onionr - Private P2P Communication.
This file contains timer control for the communicator
'''
This file contains timer control for the communicator
"""
from __future__ import annotations # thank you python, very cool
'''
import uuid
import threading
import onionrexceptions
import logger
from typing import TYPE_CHECKING
from typing import Callable, NewType, Iterable
if TYPE_CHECKING:
from communicator import OnionrCommunicatorDaemon
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@ -17,26 +27,18 @@ from __future__ import annotations # thank you python, very cool
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
"""
import uuid
import threading
import onionrexceptions, logger
from typing import TYPE_CHECKING
from typing import Callable, NewType, Iterable
from psutil import Process
if TYPE_CHECKING:
from communicator import OnionrCommunicatorDaemon
CallFreqSeconds = NewType('CallFreqSeconds', int)
class OnionrCommunicatorTimers:
def __init__(self, daemon_inst: OnionrCommunicatorDaemon,
timer_function: Callable, frequency: CallFreqSeconds,
make_thread:bool=True, thread_amount:int=1, max_threads:int=5,
requires_peer:bool=False, my_args:Iterable=[]):
make_thread: bool = True,
thread_amount: int = 1, max_threads: int = 5,
requires_peer: bool = False, my_args: Iterable = []):
self.timer_function = timer_function
self.frequency = frequency
self.thread_amount = thread_amount
@ -51,28 +53,42 @@ class OnionrCommunicatorTimers:
def processTimer(self):
# mark how many instances of a thread we have (decremented at thread end)
# mark # of instances of a thread we have (decremented at thread end)
try:
self.daemon_inst.threadCounts[self.timer_function.__name__]
except KeyError:
self.daemon_inst.threadCounts[self.timer_function.__name__] = 0
# execute thread if it is time, and we are not missing *required* online peer
# execute timer's func, if we are not missing *required* online peer
if self.count == self.frequency and not self.daemon_inst.shutdown:
try:
if self.requires_peer and len(self.daemon_inst.onlinePeers) == 0:
if self.requires_peer and \
len(self.daemon_inst.onlinePeers) == 0:
raise onionrexceptions.OnlinePeerNeeded
except onionrexceptions.OnlinePeerNeeded:
return
else:
if self.make_thread:
for i in range(self.thread_amount):
if self.daemon_inst.threadCounts[self.timer_function.__name__] >= self.max_threads:
logger.debug('%s is currently using the maximum number of threads, not starting another.' % self.timer_function.__name__)
"""
Log if a timer has max num of active threads
If this logs frequently it is indicative of a bug
or need for optimization
"""
if self.daemon_inst.threadCounts[
self.timer_function.__name__] >= \
self.max_threads:
logger.debug(
f'{self.timer_function.__name__} is currently using the maximum number of threads, not starting another.') # noqa
# if active number of threads for timer not reached yet
else:
self.daemon_inst.threadCounts[self.timer_function.__name__] += 1
newThread = threading.Thread(target=self.timer_function, args=self.args, daemon=True,
name=self.timer_function.__name__ + ' - ' + str(uuid.uuid4()))
self.daemon_inst.threadCounts[
self.timer_function.__name__] += 1
newThread = threading.Thread(
target=self.timer_function, args=self.args,
daemon=True,
name=self.timer_function.__name__ + ' - ' +
str(uuid.uuid4()))
newThread.start()
else:
self.timer_function()

16
src/communicatorutils/proxypicker.py
View File

@ -1,9 +1,9 @@
'''
Onionr - Private P2P Communication
"""
Onionr - Private P2P Communication.
Just picks a proxy to use based on a peer's address
'''
'''
Pick a proxy to use based on a peer's address
"""
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@ -16,11 +16,13 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
"""
def pick_proxy(peer_address):
if peer_address.endswith('.onion'):
return 'tor'
elif peer_address.endswith('.i2p'):
return 'i2p'
raise ValueError(f"Peer address was not string ending with acceptable value: {peer_address}")
raise ValueError(
f"Peer address not ending with acceptable domain: {peer_address}")

20
src/communicatorutils/restarttor.py
View File

@ -1,5 +1,25 @@
"""
Onionr - Private P2P Communication.
Restart Onionr managed Tor
"""
import netcontroller
import config
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def restart(comm_inst):
if not config.get('tor.use_existing_tor', False):

30
src/communicatorutils/servicecreator.py
View File

@ -1,9 +1,14 @@
'''
Onionr - Private P2P Communication
"""
Onionr - Private P2P Communication.
Creates an onionr direct connection service by scanning all connection blocks
'''
'''
Creates an onionr direct connection service by scanning all connection blocks
"""
import communicator
from onionrblocks import onionrblockapi
from onionrutils import stringvalidators, bytesconverter
from coredb import blockmetadb
from onionrservices import server_exists
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@ -16,13 +21,9 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import communicator
from onionrblocks import onionrblockapi
import logger
from onionrutils import stringvalidators, bytesconverter
from coredb import blockmetadb
from onionrservices import server_exists
"""
def service_creator(daemon):
assert isinstance(daemon, communicator.OnionrCommunicatorDaemon)
@ -30,12 +31,13 @@ def service_creator(daemon):
# TODO cache blocks and only look at recently received ones
con_blocks = blockmetadb.get_blocks_by_type('con')
for b in con_blocks:
if not b in daemon.active_services:
if b not in daemon.active_services:
bl = onionrblockapi.Block(b, decrypt=True)
bs = bytesconverter.bytes_to_str(bl.bcontent) + '.onion'
if server_exists(bl.signer):
continue
if stringvalidators.validate_pub_key(bl.signer) and stringvalidators.validate_transport(bs):
if stringvalidators.validate_pub_key(bl.signer) and \
stringvalidators.validate_transport(bs):
signer = bytesconverter.bytes_to_str(bl.signer)
daemon.active_services.append(b)
daemon.active_services.append(signer)

18
src/onionrblocks/blocklist.py
View File

@ -1,3 +1,7 @@
"""Onionr - Private P2P Communication.
Get an auto updating list of blocks
"""
from threading import Thread
from watchdog.observers import Observer
@ -7,6 +11,20 @@ from utils.identifyhome import identify_home
from coredb.dbfiles import block_meta_db
from coredb.blockmetadb import get_block_list, get_blocks_by_type
from onionrutils.epoch import get_epoch
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
class BlockList:
def __init__(self, auto_refresh=True, block_type=''):

4
src/onionrblocks/insert/main.py
View File

@ -40,7 +40,9 @@ from onionrtypes import UserIDSecretKey
def _check_upload_queue():
"""Returns the current upload queue len
"""
Return the current upload queue len.
raises OverflowError if max, false if api not running
"""
max_upload_queue: int = 5000