Compare commits

...

75 Commits

Author SHA1 Message Date
Kevin Froman 81c8c4f124 mass removal for redesigns 2021-02-22 20:56:17 +00:00
Kevin Froman 7fba65c459 work on torgossip 2021-02-21 23:22:18 +00:00
Kevin Froman 4b36e9d3da work on torgossip 2021-02-21 02:29:00 +00:00
Kevin Froman 346d30086e work on torgossip 2021-02-20 23:45:24 +00:00
Kevin Froman 60dfa8fb7e work on torgossip 2021-02-20 00:41:46 +00:00
Kevin Froman db5320124f work on torgossip 2021-02-16 05:01:01 +00:00
Kevin Froman 2ffcc2e18e work on torgossip 2021-02-15 09:11:16 +00:00
Kevin Froman 630a9b1522 work on torgossip 2021-02-13 20:22:07 +00:00
Kevin Froman 4bf1acf446 work on torgossip 2021-02-11 21:03:41 +00:00
Kevin Froman 257bef6ca0 added subprocess validator 2021-02-09 23:02:19 +00:00
Kevin Froman d9c0adcc7b work on torgossip 2021-02-08 07:32:31 +00:00
Kevin Froman 8a6129f4e2 work on torgossip 2021-02-08 01:52:12 +00:00
Kevin Froman 18d4a87973 added onion service key storage database to overcome limitations in normal torrc-based system 2021-02-07 07:25:39 +00:00
Kevin Froman ad103ee8b0 work on torgossip 2021-02-06 06:35:18 +00:00
Kevin Froman b720f2f1d5 Work on torgossip client
Use stems HS validator instead of custom logic
2021-02-05 22:16:10 +00:00
Kevin Froman bae7f745ee implemented peer announce in torgossip 2021-02-05 04:57:35 +00:00
Kevin Froman 227472d4dc implemented peer announce in torgossip 2021-02-05 04:38:57 +00:00
Kevin Froman f4182cb996 implemented peer exchange 2021-02-05 02:19:23 +00:00
Kevin Froman b4127c9836 work on torgossip 2021-02-04 05:21:37 +00:00
Kevin Froman 9d546432f5 added traceback handler 2021-02-04 01:52:36 +00:00
Kevin Froman 5ea90acd3f work on torgossip 2021-02-02 23:24:35 +00:00
Kevin Froman 9d8c7a7224 added peerDB to torgossip 2021-02-02 06:58:06 +00:00
Kevin Froman 582ac1607e remove streamfill 2021-02-02 06:57:37 +00:00
Kevin Froman c40effd520 added offset block list getter 2021-02-01 06:59:53 +00:00
Kevin Froman 9d13c0c989 work on implementing torgossip 2021-01-31 05:17:02 +00:00
Kevin Froman ee8b81ead6 work on implementing torgossip 2021-01-31 04:40:51 +00:00
Kevin Froman 418237cfc6 fix extra onion icon displaying in stat panel 2021-01-29 23:54:14 +00:00
Kevin Froman 84987cfc21 added unsafe socks denial to torrc generator 2021-01-29 21:30:58 +00:00
Kevin Froman 63f7209bdd Added blockcreatorqueue and subprocess generator for new block system 2021-01-29 21:15:18 +00:00
Kevin Froman 67d9dc3557 added more modules to dependencycheck.py 2021-01-29 20:57:39 +00:00
Kevin Froman a2da6b8c89 added serialized api work 2021-01-27 18:15:06 +00:00
Kevin Froman 6625b7ce19 added serialized api work 2021-01-27 08:38:45 +00:00
Kevin Froman da353476cf remove floodfill script 2021-01-27 07:12:05 +00:00
Kevin Froman 828c78a569 work on torgossip, added base64 code whitelist because of a werid plugin issue with bigbrother 2021-01-27 05:34:31 +00:00
Kevin Froman a55055e720 started work on new gossip streaming protocol 2021-01-26 05:39:23 +00:00
Kevin Froman ba8ba6d3d8 properly clean block list in blockio 2021-01-25 22:16:51 +00:00
Kevin Froman 708c5d2e71 added block cleaning method for new database 2021-01-25 06:28:18 +00:00
Kevin Froman 75234310ca added blockio load implementation 2021-01-24 23:29:02 +00:00
Kevin Froman 5ed5b8455a bump blocks version 2021-01-24 23:28:25 +00:00
Kevin Froman 003718db04 Merge branch 'master' into integrate-new-blocks 2021-01-24 18:11:42 +00:00
Kevin Froman db96f6ad7e bump blocks version 2021-01-24 07:15:27 +00:00
Kevin Froman 7cb4731156 Fix scroll jumping by changing connected node list to an iframe 2021-01-21 22:17:05 +00:00
Kevin Froman d03620db7e merge after rebasing toast fix 2021-01-21 20:26:41 +00:00
Kevin Froman 1d1edfd0f1 work on blockio api 2021-01-21 20:23:59 +00:00
Kevin Froman 001bdda433 reload config before spawning daemon threads 2021-01-21 20:23:58 +00:00
Kevin Froman aeb2e16d8b bump dep version for faster vdf 2021-01-21 20:23:58 +00:00
Kevin Froman 847f80b8f4 implemented fetch block in blockio 2021-01-21 20:23:58 +00:00
Kevin Froman 2ffc8c637e fix broken non protected safedb get 2021-01-21 20:23:58 +00:00
Kevin Froman e41e3fd01d bump onionrblocks for missing generator module 2021-01-21 20:23:58 +00:00
Kevin Froman a3ce191ed3 renamed old block api due to namespace conflict
fix onboarding not using optimize cpu option
2021-01-21 20:23:58 +00:00
Kevin Froman 9e2d374270 bump onionrblocks for missing kasten dependency 2021-01-21 20:23:58 +00:00
Kevin Froman 504ef3a49a work on removing communicator 2021-01-21 20:23:58 +00:00
Kevin Froman ec9dbe383c added blockio doc for db keys 2021-01-21 20:23:58 +00:00
Kevin Froman 7390539d50 added blockio module directory 2021-01-21 20:23:58 +00:00
Kevin Froman 0f7ea4e4c8 added license boilerplate to safedb 2021-01-21 20:23:58 +00:00
Kevin Froman 86dca2dd2b added work on faster and safer database for blocks and general KV 2021-01-21 20:23:58 +00:00
Kevin Froman 0e69d88708 added put/get from db 2021-01-21 20:23:58 +00:00
Kevin Froman cd0ec77189 added work on faster and safer database for blocks and general KV 2021-01-21 20:23:58 +00:00
Kevin Froman 6bcfd91352 bump dependencies 2021-01-21 20:23:58 +00:00
Kevin Froman 5df747d658 added work on faster and safer database for blocks and general KV 2021-01-21 20:23:58 +00:00
Kevin Froman b7ab487fac fixed toast spam #29 2021-01-21 20:23:36 +00:00
Kevin Froman f44ca60c6e implemented fetch block in blockio 2021-01-17 20:50:32 +00:00
Kevin Froman b6bb350b65 fix broken non protected safedb get 2021-01-17 19:44:24 +00:00
Kevin Froman 30b73c7f7c bump onionrblocks for missing generator module 2021-01-17 19:44:00 +00:00
Kevin Froman ca37f6df9d renamed old block api due to namespace conflict
fix onboarding not using optimize cpu option
2021-01-17 04:58:28 +00:00
Kevin Froman f5dfd16408 bump onionrblocks for missing kasten dependency 2021-01-17 03:59:41 +00:00
Kevin Froman 9ed17f4fb6 work on removing communicator 2021-01-17 00:40:04 +00:00
Kevin Froman dc51c26b13 added blockio doc for db keys 2021-01-17 00:39:32 +00:00
Kevin Froman db641341c5 added blockio module directory 2021-01-16 20:45:59 +00:00
Kevin Froman 8ddc38515d added license boilerplate to safedb 2021-01-16 20:45:59 +00:00
Kevin Froman 7ab330a710 added work on faster and safer database for blocks and general KV 2021-01-16 20:45:59 +00:00
Kevin Froman 153d530f38 added put/get from db 2021-01-16 20:45:58 +00:00
Kevin Froman 9e01c65c0f added work on faster and safer database for blocks and general KV 2021-01-16 20:45:58 +00:00
Kevin Froman 2eb3e93666 bump dependencies 2021-01-16 20:45:58 +00:00
Kevin Froman f1edd1932e added work on faster and safer database for blocks and general KV 2021-01-16 20:45:58 +00:00
337 changed files with 4241 additions and 14515 deletions

View File

@ -21,7 +21,7 @@
| [Docs](#documentation)/[web copy](https://beardog108.github.io/onionr/) | [Get involved](#help-out) | [Onionr.net](https://onionr.net/)/[.onion](http://onionrbak72t5zhbzuey2fdkpczlvhowgcpqc6uoyrd3uxztzxwz5cyd.onion/) |
<hr>
---
**The main repository for this software is at https://git.VoidNet.tech/kev/onionr/**
@ -30,20 +30,26 @@ Mirrors: [Github](https://github.com/beardog108/onionr), [Gitlab](https://gitlab
Onionr ("Onion Relay") is a decentralized/distributed peer-to-peer communication network, designed to be anonymous and resistant to (meta)data analysis, spam, and corruption.
Onionr stores data in independent packages referred to as 'blocks'. The blocks are distributed to all interested nodes. Blocks and user IDs cannot be easily proven to have been created by a particular user. Even if there is enough evidence to believe that a specific user created a block, nodes still operate behind Tor and as such cannot be trivially unmasked. Anonymity is achieved by a stateless network, with no given indication of what node a block originates from. In fact, since one is not required to participate in routing or storage to insert a message, blocks often do not originate from any identifiable node.
Onionr gives the individual the ability to speak freely, without fear of surveillance and censorship.
---
Onionr stores data in independent packages referred to as 'blocks'. The blocks are distributed to all nodes interested in their data type. Blocks and user IDs cannot be easily proven to have been created by a particular user. Even if there is enough evidence to believe that a specific user created a block, nodes still operate behind Tor and as such cannot be trivially unmasked. Anonymity is achieved by a stateless network, with no given indication of what node a block originates from. In fact, since one is not required to participate in routing or storage to insert a message, blocks often do not originate from any identifiable node.
Onionr works primarily via epidemic/gossip style routing, with message delivery taking roughly log<sub>F</sub>(N) cycles where F is the number of nodes to send a message to each cycle and N is the number of connected nodes. So a network of 100 million nodes can deliver messages in a few minutes even with high packet loss and malfunctioning nodes.
Through message mixing and key privacy, it is intended to be nigh impossible to discover the identity of a message creator or recipient. Via long-term traffic analysis, a well funded adversary may discover the most probable node(s) to be creating a set of related blocks, however doing so would only lead them to a node behind Tor. As the first node that a block appears on is almost always not the creator of the block, there is plausible deniability regarding the true creator of the block.
Onionr gives the individual the ability to speak freely, without fear of surveillance and censorship.
Users are identified by ed25519/curve25519 public keys, which can be used to sign blocks or send encrypted data.
Onionr can be used for mail, as a social network, instant messenger, file sharing software, or for encrypted group discussion.
Due to the nature of anonymity, the graph as implemented in this reference network is dense, undirected, cyclic and can be disconnected. As a result, current scalability is poor but sufficient for high latency communications. As the need arises isolated stream solutions may be implemented (in a manner similar to described in the Bitmessage whitepaper). Since Onionr is technically just a data format, any routing scheme can be used to pass messages.
Due to the nature of anonymity, the graph as implemented in this reference network is dense, undirected, cyclic and can be disconnected. Since Onionr is technically just a data format, any routing scheme can be used to pass messages.
The whitepaper is available [here](docs/whitepaper.md).
---
## Main Features
* [X] 🌐 Fully p2p/decentralized, no trackers or other single points of failure
@ -57,9 +63,9 @@ Onionr ships with various application plugins ready for use out of the box:
Currently usable:
* Mail
* Public anonymous chat/message board
* Simple webpage hosting - Will be greatly extended
* 📨 Mail
* 💬 Public anonymous chat/message board
* 📃 Simple webpage hosting - Will be greatly extended
* File sharing (Work in progress)
Not yet usable:
@ -154,9 +160,9 @@ Donating at least $3 gets you cool Onionr stickers. Get in touch if you want the
* Monero: 4B5BA24d1P3R5aWEpkGY5TP7buJJcn2aSGBVRQCHhpiahxeB4aWsu15XwmuTjC6VF62NApZeJGTS248RMVECP8aW73Uj2ax
* USD (Card/Paypal): [Ko-Fi](https://www.ko-fi.com/beardogkf)
* USD (Card/Paypal (no account required)): [Ko-Fi](https://www.ko-fi.com/beardogkf)
* [Indiegogo](https://igg.me/at/onionr/x#/)
* Sign up for [privacy.com (refferal link)](https://privacy.com/join/FNNDF) to protect your personal information when contributing or shopping elsewhere, we both get $5 USD.
Note: probably not tax deductible

View File

@ -1,18 +0,0 @@
* add GUI config editor
* add multi-device forward secrecy
* document anonymity & security theory
* document usage
* ensure accessibility for Onionr web UI
* make forward secrecy compatible with multiple devices
* add way to mark key as dead
* add hashable set password for web ui
* add edits to circles posts
* make node "speed" setting such as when ui is open to reduce bandwidth usage
* localization support
* add BCC support to mail
* truncate last N blocks when sharing list

View File

@ -0,0 +1,6 @@
Blockio wraps safedb, the new key value database module
This is how the keys are setup and what they are for:
bl-{type}: bytes of hashes for each block type
{block hash}: block data

View File

@ -64,18 +64,6 @@ Please note: endpoints that simply provide static web app files are not document
* /getuptime
- Methods: GET
- Returns uptime in seconds
* /getActivePubkey
- Methods: GET
- Returns the current active public key in base32 format
* /getHumanReadable/pubkey
- Methods: GET
- Echos the specified public key in mnemonic format
* /insertblock
- Methods: POST
- Accepts JSON data for creating a new block. 'message' contains the block data, 'to' specifies the peer's public key to encrypt the data to, 'sign' is a boolean for signing the message.
* /torready
- Methods: POST
- Returns boolean if Tor is started or not
# Public API

View File

@ -7,3 +7,4 @@ A paper being listed here is not end-all-be-all endorsement of every detail insi
* [Protecting Free Expression Online with Freenet](https://freenetproject.org/assets/papers/ddisrs.pdf)
* [Bitmessage: A PeertoPeer Message Authentication and Delivery System](https://archive.org/details/BitmessageWhitepaper/)
* [MuON: Epidemic based Mutual Anonymity](https://web.archive.org/web/20060901153544/http://www.csl.mtu.edu/cs6461/www/Reading/MuON_ICNP2005.pdf)
* [SWIM: Scalable Weakly-consistent Infection-style Process Group Membership Protocol](https://www.cs.cornell.edu/projects/Quicksilver/public_pdfs/SWIM.pdf)

View File

@ -1,7 +1,7 @@
urllib3==1.25.11
requests==2.25.1
PyNaCl==1.4.0
gevent==20.9.0
gevent==20.12.1
Flask==1.1.2
PySocks==1.7.1
stem==1.8.0
@ -14,4 +14,5 @@ psutil==5.8.0
filenuke==0.0.0
watchdog==1.0.2
ujson==4.0.1
cffi==1.14.4
cffi==1.14.4
onionrblocks==4.1.0

View File

@ -2,11 +2,11 @@
# This file is autogenerated by pip-compile
# To update, run:
#
# pip-compile --generate-hashes requirements.in
# pip-compile --generate-hashes --output-file=requirements.txt requirements.in
#
certifi==2018.11.29 \
--hash=sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7 \
--hash=sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033 \
--hash=sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033
# via requests
cffi==1.14.4 \
--hash=sha256:00a1ba5e2e95684448de9b89888ccd02c98d512064b4cb987d48f4b40aa0421e \
@ -44,39 +44,55 @@ cffi==1.14.4 \
--hash=sha256:ec80dc47f54e6e9a78181ce05feb71a0353854cc26999db963695f950b5fb375 \
--hash=sha256:f032b34669220030f905152045dfa27741ce1a6db3324a5bc0b96b6c7420c87b \
--hash=sha256:f60567825f791c6f8a592f3c6e3bd93dd2934e3f9dac189308426bd76b00ef3b \
--hash=sha256:f803eaa94c2fcda012c047e62bc7a51b0bdabda1cad7a92a522694ea2d76e49f \
# via -r requirements.in, pynacl
--hash=sha256:f803eaa94c2fcda012c047e62bc7a51b0bdabda1cad7a92a522694ea2d76e49f
# via
# -r requirements.in
# pynacl
chardet==3.0.4 \
--hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \
--hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \
--hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691
# via requests
click==7.0 \
--hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \
--hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 \
--hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7
# via flask
deadsimplekv==0.3.2 \
--hash=sha256:a725f4a9d1156ebb66b7535ac150006881e0365b715e34e3709214827b8b0c4c \
--hash=sha256:df00262d26c3dcfecb710425a7413059480d8cf026216042d7cbffb8514818b2 \
--hash=sha256:df00262d26c3dcfecb710425a7413059480d8cf026216042d7cbffb8514818b2
# via -r requirements.in
filenuke==0.0.0 \
--hash=sha256:147011c0125121469cae0a8a7f4df399f470e54aa29a08f2d2c099bf0118dcee \
--hash=sha256:c55535dcecfdb27c5f4ce664d46e115950b5429763b5db75c198053646177f8f \
--hash=sha256:c55535dcecfdb27c5f4ce664d46e115950b5429763b5db75c198053646177f8f
# via -r requirements.in
flask==1.1.2 \
--hash=sha256:4efa1ae2d7c9865af48986de8aeb8504bf32c7f3d6fdc9353d34b21f4b127060 \
--hash=sha256:8a4fdd8936eba2512e9c85df320a37e694c93945b33ef33c89946a340a238557 \
--hash=sha256:8a4fdd8936eba2512e9c85df320a37e694c93945b33ef33c89946a340a238557
# via -r requirements.in
gevent==20.9.0 \
--hash=sha256:1628a403fc9c3ea9b35924638a4d4fbe236f60ecdf4e22ed133fbbaf0bc7cb6b \
--hash=sha256:2269574444113cb4ca1c1808ab9460a87fe25e1c34a6e36d975d4af46e4afff9 \
--hash=sha256:324808a8558c733f7a9734525483795d52ca3bbd5662b24b361d81c075414b1f \
--hash=sha256:5f6d48051d336561ec08995431ee4d265ac723a64bba99cc58c3eb1a4d4f5c8d \
--hash=sha256:a8733a01974433d91308f8c44fa6cc13428b15bb39d46540657e260ff8852cb1 \
--hash=sha256:adbb267067f56696b2babced3d0856aa39dcf14b8ccd2dffa1fab587b00c6f80 \
--hash=sha256:b07fcbca3e819296979d82fac3d8b44f0d5ced57b9a04dffcfd194da99c8eb2d \
--hash=sha256:b2948566003a1030e47507755fe1f446995e8671c0c67571091539e01faf94cc \
--hash=sha256:e11de4b4d107ca2f35000eb08e9c4c4621c153103b400f48a9ea95b96d8c7e0b \
--hash=sha256:fb33dc1ab27557bccd64ad4bf81e68c8b0d780fe937b1e2c0814558798137229 \
gevent==20.12.1 \
--hash=sha256:0f9fa230c5878704b9e286ad5038bac3b70d293bf10e9efa8b2ae1d7d80e7e08 \
--hash=sha256:19bd3fe60dec45fe6420b7772496950215f1b36701905876ba1644b6b2064163 \
--hash=sha256:2d05f38a5ef1ebb7ceb692897674b11ba603914524765b989c65c020c7b08360 \
--hash=sha256:4b0a5626c4e534d184cdf00d66f06de3885beafaaa5f7b98d47186ea175629a1 \
--hash=sha256:4baecba0fd614e14dc1f3f8c35616cb248cdb893de576150ed1fc7fc66b8ba3d \
--hash=sha256:60799fd7dcbb622f8435eb12436d48a8d27f8e7b3d23631e32ccc04ddd2097c2 \
--hash=sha256:69ddc1767a02f68e71d5e0d3215aa4d28872187715627f71ff0eadd7b7a5e7f4 \
--hash=sha256:7a808c63f065a303bbbe87c5c0754e06abb1e23e18752f418dce1eb3189cb43d \
--hash=sha256:81e38ed46e21e0b00b930efce1a1ff46c7722ad83d84052f71a757f23cbed1c0 \
--hash=sha256:895c76a89907d9d37fdfaf5321cb0fff0cba396f003bedb4f5fc13836da6f250 \
--hash=sha256:89c583744f91052ae987356660f5ed0b8fc59a1230b051d6ccc10d37a155fe01 \
--hash=sha256:99b68765767bb3e2244a66b012883899a6f17c23b6dc1cd80b793df341e15f08 \
--hash=sha256:9d001fc899db6e140110ae7484e58cd74b0dfa5cee021a0347f00bb441ac78bd \
--hash=sha256:b57586ad3fedf13d351d2559b70d6fe593c50400315d52bb3c072285da60fa37 \
--hash=sha256:ba244028225ff8d3a58f344fcd16ab05b0e3642b34d81f51f7fa3c70761f6c34 \
--hash=sha256:bf946a99e364ebcc95b82c794d5d1a67f13115adbefab7b9e12791f13184cfd5 \
--hash=sha256:c3706a620e167c4bd007f16f113928324c4e07a7bae11d6d18d65f82abcd7a58 \
--hash=sha256:c570a2e3100f758a5c2f9b993ecf870ee784390e44e1a292c361d6b32fb3ad4c \
--hash=sha256:caec00914e8f21b2c77a29bbc2ef3abfeadf7515656e5451dfb14c2064733998 \
--hash=sha256:e233ae153b586b61e492806d4cd1be2217de7441922c02053b67de14800bce96 \
--hash=sha256:f020bfb34d57caa10029111be776524c378a4aac8417bc6fb1154b05e00fc220 \
--hash=sha256:f3faf1834464f1b0731aa6346cd9f41029fa9e208d6ecbce4a736c19562c86aa \
--hash=sha256:f857adbe1bf41e620d86173a53100f4ec328eba3089069a4815b3d9f4229dee8 \
--hash=sha256:ffa1be13963db6aa55c50d2fd4a656c82f53a03a47e37aaa69e79a488123538d
# via -r requirements.in
greenlet==0.4.17 \
--hash=sha256:1023d7b43ca11264ab7052cb09f5635d4afdb43df55e0854498fc63070a0b206 \
@ -96,20 +112,24 @@ greenlet==0.4.17 \
--hash=sha256:ccd62f09f90b2730150d82f2f2ffc34d73c6ce7eac234aed04d15dc8a3023994 \
--hash=sha256:d3436110ca66fe3981031cc6aff8cc7a40d8411d173dde73ddaa5b8445385e2d \
--hash=sha256:e495096e3e2e8f7192afb6aaeba19babc4fb2bdf543d7b7fed59e00c1df7f170 \
--hash=sha256:e66a824f44892bc4ec66c58601a413419cafa9cec895e63d8da889c8a1a4fa4a \
--hash=sha256:e66a824f44892bc4ec66c58601a413419cafa9cec895e63d8da889c8a1a4fa4a
# via gevent
idna==2.7 \
--hash=sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e \
--hash=sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16 \
--hash=sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16
# via requests
itsdangerous==1.1.0 \
--hash=sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19 \
--hash=sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749 \
--hash=sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749
# via flask
jinja2==2.11.1 \
--hash=sha256:93187ffbc7808079673ef52771baa950426fd664d3aad1d0fa3e95644360e250 \
--hash=sha256:b0eaf100007721b5c16c1fc1eecb87409464edc10469ddc9a22a27a99123be49 \
--hash=sha256:b0eaf100007721b5c16c1fc1eecb87409464edc10469ddc9a22a27a99123be49
# via flask
kasten==3.0.0 \
--hash=sha256:52894af46d6e1339f0d5fa8961892b292f99176848bce11877fe4a435b6782e5 \
--hash=sha256:b22ebdc5f475c2ef9ab74abc36552add0b37732a7ce2be6bd7977ee41b2163b4
# via onionrblocks
markupsafe==1.1.1 \
--hash=sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473 \
--hash=sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161 \
@ -143,11 +163,48 @@ markupsafe==1.1.1 \
--hash=sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f \
--hash=sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2 \
--hash=sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7 \
--hash=sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be \
--hash=sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be
# via jinja2
mimcvdf==1.2.0 \
--hash=sha256:647009b6f13173ac84a683b491566c8b63bc3dd5d60edd1e109b41cd311e2c08
# via kasten
msgpack==1.0.2 \
--hash=sha256:0cb94ee48675a45d3b86e61d13c1e6f1696f0183f0715544976356ff86f741d9 \
--hash=sha256:1026dcc10537d27dd2d26c327e552f05ce148977e9d7b9f1718748281b38c841 \
--hash=sha256:26a1759f1a88df5f1d0b393eb582ec022326994e311ba9c5818adc5374736439 \
--hash=sha256:2a5866bdc88d77f6e1370f82f2371c9bc6fc92fe898fa2dec0c5d4f5435a2694 \
--hash=sha256:31c17bbf2ae5e29e48d794c693b7ca7a0c73bd4280976d408c53df421e838d2a \
--hash=sha256:497d2c12426adcd27ab83144057a705efb6acc7e85957a51d43cdcf7f258900f \
--hash=sha256:5a9ee2540c78659a1dd0b110f73773533ee3108d4e1219b5a15a8d635b7aca0e \
--hash=sha256:8521e5be9e3b93d4d5e07cb80b7e32353264d143c1f072309e1863174c6aadb1 \
--hash=sha256:87869ba567fe371c4555d2e11e4948778ab6b59d6cc9d8460d543e4cfbbddd1c \
--hash=sha256:8ffb24a3b7518e843cd83538cf859e026d24ec41ac5721c18ed0c55101f9775b \
--hash=sha256:92be4b12de4806d3c36810b0fe2aeedd8d493db39e2eb90742b9c09299eb5759 \
--hash=sha256:9ea52fff0473f9f3000987f313310208c879493491ef3ccf66268eff8d5a0326 \
--hash=sha256:a4355d2193106c7aa77c98fc955252a737d8550320ecdb2e9ac701e15e2943bc \
--hash=sha256:a99b144475230982aee16b3d249170f1cccebf27fb0a08e9f603b69637a62192 \
--hash=sha256:ac25f3e0513f6673e8b405c3a80500eb7be1cf8f57584be524c4fa78fe8e0c83 \
--hash=sha256:b28c0876cce1466d7c2195d7658cf50e4730667196e2f1355c4209444717ee06 \
--hash=sha256:b55f7db883530b74c857e50e149126b91bb75d35c08b28db12dcb0346f15e46e \
--hash=sha256:b6d9e2dae081aa35c44af9c4298de4ee72991305503442a5c74656d82b581fe9 \
--hash=sha256:c747c0cc08bd6d72a586310bda6ea72eeb28e7505990f342552315b229a19b33 \
--hash=sha256:d6c64601af8f3893d17ec233237030e3110f11b8a962cb66720bf70c0141aa54 \
--hash=sha256:d8167b84af26654c1124857d71650404336f4eb5cc06900667a493fc619ddd9f \
--hash=sha256:de6bd7990a2c2dabe926b7e62a92886ccbf809425c347ae7de277067f97c2887 \
--hash=sha256:e36a812ef4705a291cdb4a2fd352f013134f26c6ff63477f20235138d1d21009 \
--hash=sha256:e89ec55871ed5473a041c0495b7b4e6099f6263438e0bd04ccd8418f92d5d7f2 \
--hash=sha256:f3e6aaf217ac1c7ce1563cf52a2f4f5d5b1f64e8729d794165db71da57257f0c \
--hash=sha256:f484cd2dca68502de3704f056fa9b318c94b1539ed17a4c784266df5d6978c87 \
--hash=sha256:fae04496f5bc150eefad4e9571d1a76c55d021325dcd484ce45065ebbdd00984 \
--hash=sha256:fe07bc6735d08e492a327f496b7850e98cb4d112c56df69b0c844dbebcbb47f6
# via kasten
niceware==0.2.1 \
--hash=sha256:0f8b192f2a1e800e068474f6e208be9c7e2857664b33a96f4045340de4e5c69c \
--hash=sha256:cf2dc0e1567d36d067c61b32fed0f1b9c4534ed511f9eeead4ba548d03b5c9eb \
--hash=sha256:cf2dc0e1567d36d067c61b32fed0f1b9c4534ed511f9eeead4ba548d03b5c9eb
# via -r requirements.in
onionrblocks==4.1.0 \
--hash=sha256:2f806d1a4cf332ffef8630ac3d362499854316d957945be0a090c7ff6917a6c4 \
--hash=sha256:bfdfa90df6fcdaef44b9ff1bc8f7b645fc41d10cf46481a3158d6c77e3832507
# via -r requirements.in
psutil==5.8.0 \
--hash=sha256:0066a82f7b1b37d334e68697faba68e5ad5e858279fd6351c8ca6024e8d6ba64 \
@ -177,10 +234,10 @@ psutil==5.8.0 \
--hash=sha256:ea313bb02e5e25224e518e4352af4bf5e062755160f77e4b1767dd5ccb65f876 \
--hash=sha256:ea372bcc129394485824ae3e3ddabe67dc0b118d262c568b4d2602a7070afdb0 \
--hash=sha256:f4634b033faf0d968bb9220dd1c793b897ab7f1189956e1aa9eae752527127d3 \
--hash=sha256:fcc01e900c1d7bee2a37e5d6e4f9194760a93597c97fee89c4ae51701de03563 \
--hash=sha256:fcc01e900c1d7bee2a37e5d6e4f9194760a93597c97fee89c4ae51701de03563
# via -r requirements.in
pycparser==2.19 \
--hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3 \
--hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3
# via cffi
pynacl==1.4.0 \
--hash=sha256:06cbb4d9b2c4bd3c8dc0d267416aaed79906e7b33f114ddbf0911969794b1cc4 \
@ -198,30 +255,34 @@ pynacl==1.4.0 \
--hash=sha256:cd401ccbc2a249a47a3a1724c2918fcd04be1f7b54eb2a5a71ff915db0ac51c6 \
--hash=sha256:d452a6746f0a7e11121e64625109bc4468fc3100452817001dbe018bb8b08514 \
--hash=sha256:ea6841bc3a76fa4942ce00f3bda7d436fda21e2d91602b9e21b7ca9ecab8f3ff \
--hash=sha256:f8851ab9041756003119368c1e6cd0b9c631f46d686b3904b18c0139f4419f80 \
# via -r requirements.in
--hash=sha256:f8851ab9041756003119368c1e6cd0b9c631f46d686b3904b18c0139f4419f80
# via
# -r requirements.in
# onionrblocks
pysocks==1.7.1 \
--hash=sha256:08e69f092cc6dbe92a0fdd16eeb9b9ffbc13cadfe5ca4c7bd92ffb078b293299 \
--hash=sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5 \
--hash=sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0 \
--hash=sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0
# via -r requirements.in
requests==2.25.1 \
--hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804 \
--hash=sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e \
# via -r requirements.in, streamedrequests
--hash=sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e
# via
# -r requirements.in
# streamedrequests
six==1.12.0 \
--hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \
--hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \
--hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73
# via pynacl
stem==1.8.0 \
--hash=sha256:a0b48ea6224e95f22aa34c0bc3415f0eb4667ddeae3dfb5e32a6920c185568c2 \
--hash=sha256:a0b48ea6224e95f22aa34c0bc3415f0eb4667ddeae3dfb5e32a6920c185568c2
# via -r requirements.in
streamedrequests==1.0.3 \
--hash=sha256:4388ffc0ee94dda719dafc4324b8ddd108cb2231ec59871de79e2592bf4eef0a \
--hash=sha256:ee68417a1522e75c35b1b2d5f3b6f7e76a3a1a6c0ef5e0c573d08307910079d8 \
--hash=sha256:ee68417a1522e75c35b1b2d5f3b6f7e76a3a1a6c0ef5e0c573d08307910079d8
# via -r requirements.in
toomanyobjs==1.1.0 \
--hash=sha256:99e27468f9dad19127be9e2fb086b42acd69aed9ad7e63cef74d6e4389be0534 \
--hash=sha256:99e27468f9dad19127be9e2fb086b42acd69aed9ad7e63cef74d6e4389be0534
# via -r requirements.in
ujson==4.0.1 \
--hash=sha256:078808c385036cba73cad96f498310c61e9b5ae5ac9ea01e7c3996ece544b556 \
@ -244,16 +305,18 @@ ujson==4.0.1 \
--hash=sha256:c354c1617b0a4378b6279d0cd511b769500cf3fa7c42e8e004cbbbb6b4c2a875 \
--hash=sha256:c604024bd853b5df6be7d933e934da8dd139e6159564db7c55b92a9937678093 \
--hash=sha256:e7ab24942b2d57920d75b817b8eead293026db003247e26f99506bdad86c61b4 \
--hash=sha256:f8a60928737a9a47e692fcd661ef2b5d75ba22c7c930025bd95e338f2a6e15bc \
--hash=sha256:f8a60928737a9a47e692fcd661ef2b5d75ba22c7c930025bd95e338f2a6e15bc
# via -r requirements.in
unpaddedbase32==0.2.0 \
--hash=sha256:4aacee75f8fd6c8cf129842ecba45ca59c11bfb13dae19d86f32b48fa3715403 \
--hash=sha256:b7b780c31d27d55e66abf6c221216a35690ee8892c2daacff7f2528e229bd9c3 \
--hash=sha256:b7b780c31d27d55e66abf6c221216a35690ee8892c2daacff7f2528e229bd9c3
# via -r requirements.in
urllib3==1.25.11 \
--hash=sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2 \
--hash=sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e \
# via -r requirements.in, requests
--hash=sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e
# via
# -r requirements.in
# requests
watchdog==1.0.2 \
--hash=sha256:016b01495b9c55b5d4126ed8ae75d93ea0d99377084107c33162df52887cee18 \
--hash=sha256:101532b8db506559e52a9b5d75a308729b3f68264d930670e6155c976d0e52a0 \
@ -271,15 +334,15 @@ watchdog==1.0.2 \
--hash=sha256:e7c73edef48f4ceeebb987317a67e0080e5c9228601ff67b3c4062fa020403c7 \
--hash=sha256:ee21aeebe6b3e51e4ba64564c94cee8dbe7438b9cb60f0bb350c4fa70d1b52c2 \
--hash=sha256:f1d0e878fd69129d0d68b87cee5d9543f20d8018e82998efb79f7e412d42154a \
--hash=sha256:f84146f7864339c8addf2c2b9903271df21d18d2c721e9a77f779493234a82b5 \
--hash=sha256:f84146f7864339c8addf2c2b9903271df21d18d2c721e9a77f779493234a82b5
# via -r requirements.in
werkzeug==0.15.5 \
--hash=sha256:87ae4e5b5366da2347eb3116c0e6c681a0e939a33b2805e2c0cbd282664932c4 \
--hash=sha256:a13b74dd3c45f758d4ebdb224be8f1ab8ef58b3c0ffc1783a8c7d9f4f50227e6 \
--hash=sha256:a13b74dd3c45f758d4ebdb224be8f1ab8ef58b3c0ffc1783a8c7d9f4f50227e6
# via flask
zope.event==4.4 \
--hash=sha256:69c27debad9bdacd9ce9b735dad382142281ac770c4a432b533d6d65c4614bcf \
--hash=sha256:d8e97d165fd5a0997b45f5303ae11ea3338becfe68c401dd88ffd2113fe5cae7 \
--hash=sha256:d8e97d165fd5a0997b45f5303ae11ea3338becfe68c401dd88ffd2113fe5cae7
# via gevent
zope.interface==5.1.0 \
--hash=sha256:0103cba5ed09f27d2e3de7e48bb320338592e2fabc5ce1432cf33808eb2dfd8b \
@ -321,7 +384,7 @@ zope.interface==5.1.0 \
--hash=sha256:ef739fe89e7f43fb6494a43b1878a36273e5924869ba1d866f752c5812ae8d58 \
--hash=sha256:f40db0e02a8157d2b90857c24d89b6310f9b6c3642369852cdc3b5ac49b92afc \
--hash=sha256:f68bf937f113b88c866d090fea0bc52a098695173fc613b055a17ff0cf9683b6 \
--hash=sha256:fb55c182a3f7b84c1a2d6de5fa7b1a05d4660d866b91dbf8d74549c57a1499e8 \
--hash=sha256:fb55c182a3f7b84c1a2d6de5fa7b1a05d4660d866b91dbf8d74549c57a1499e8
# via gevent
# WARNING: The following packages were not pinned, but pip requires them to be

View File

@ -11,7 +11,7 @@ import os
if not os.path.exists('onionr.sh'):
os.chdir('../')
sys.path.append("src/")
import onionrblocks
import oldblocks
amount = int(input("Number of blocks:"))
@ -24,8 +24,8 @@ else:
for i in range(amount):
if expire:
print(onionrblocks.insert(data=os.urandom(32), expire=expire))
print(oldblocks.insert(data=os.urandom(32), expire=expire))
else:
print(onionrblocks.insert(data=os.urandom(32)))
print(oldblocks.insert(data=os.urandom(32)))
print(i, "done")

View File

@ -1,21 +0,0 @@
import sys
import os
import subprocess
import base64
if not os.path.exists('onionr.sh'):
os.chdir('../')
sys.path.append("src/")
from streamfill import identify_neighbors
onions = []
p = subprocess.Popen(["scripts/generate-onions.py", '5'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
for line in iter(p.stdout.readline, b''):
line = line.decode()
onions.append(line.strip())
for onion in onions:
print(onion, identify_neighbors(onion, onions, 3))

View File

@ -6,27 +6,14 @@ import json
conf = json.load(open('static-data/default_config.json', 'r'))
conf['tor']['use_existing_tor'] = False
conf['tor']['existing_control_port'] = 0
conf['tor']['existing_control_password'] = ""
conf['tor']['existing_socks_port'] = 0
conf['general']['dev_mode'] = False
conf['general']['insert_deniable_blocks'] = True
conf['general']['random_bind_ip'] = True
conf['general']['display_header'] = True
conf['general']['security_level'] = 0
conf['general']['use_bootstrap_list'] = True
conf['onboarding']['done'] = False
conf['general']['minimum_block_pow'] = 5
conf['general']['minimum_send_pow'] = 5
conf['log']['file']['remove_on_exit'] = True
conf['transports']['lan'] = True
conf['transports']['tor'] = True
conf['transports']['sneakernet'] = True
conf['statistics']['i_dont_want_privacy'] = False
conf['statistics']['server'] = ''
conf['ui']['animated_background'] = True
conf['runtests']['skip_slow'] = False
json.dump(conf, open('static-data/default_config.json', 'w'), sort_keys=True, indent=4)

View File

@ -8,32 +8,16 @@ input("enter to continue") # hack to avoid vscode term input
conf = json.load(open('static-data/default_config.json', 'r'))
block_pow = int(input("Block POW level:"))
conf['general']['security_level'] = int(input("Security level:"))
conf['transports']['tor'] = False
if input('Use Tor? y/n').lower() == 'y':
conf['transports']['tor'] = True
if input("Reuse Tor? y/n:").lower() == 'y':
conf['tor']['use_existing_tor'] = True
conf['tor']['existing_control_port'] = int(input("Enter existing control port:"))
conf['tor']['existing_control_password'] = input("Tor pass:")
conf['tor']['existing_socks_port'] = int(input("Existing socks port:"))
conf['general']['dev_mode'] = True
conf['general']['insert_deniable_blocks'] = False
conf['general']['random_bind_ip'] = False
conf['onboarding']['done'] = True
conf['general']['minimum_block_pow'] = block_pow
conf['general']['minimum_send_pow'] = block_pow
conf['general']['use_bootstrap_list'] = False
if input("Use bootstrap list? y/n").lower() == 'y':
conf['general']['use_bootstrap_list'] = True
conf['log']['file']['remove_on_exit'] = False
conf['ui']['animated_background'] = False
if input('Stat reporting? y/n') == 'y':
conf['statistics']['i_dont_want_privacy'] = True
conf['statistics']['server'] = input('Statistics server')
conf['runtests']['skip_slow'] = True
json.dump(conf, open('static-data/default_config.json', 'w'), sort_keys=True, indent=4)

View File

@ -1,33 +0,0 @@
import networkx as nx
import matplotlib.pyplot as plt
import sys
import os
import subprocess
import base64
if not os.path.exists('onionr.sh'):
os.chdir('../')
sys.path.append("src/")
from streamfill import identify_neighbors
G = nx.Graph()
size = 1000
onions = []
p = subprocess.Popen(["scripts/generate-onions.py", str(size)],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
for line in iter(p.stdout.readline, b''):
line = line.decode().strip()
onions.append(line)
G.add_node(line[:6])
for onion in onions:
neighbors = identify_neighbors(onion, onions, 0.15 * size)
for neighbor in neighbors:
G.add_edge(onion[:6], neighbor[:6])
#nx.draw(G, with_labels=True, font_weight='bold')
#nx.draw_shell(G, with_labels=True)
#nx.draw_random(G, with_labels=True)
nx.draw_kamada_kawai(G)
plt.savefig("graph.png")

View File

@ -2,40 +2,21 @@
import sys
import os
import stem
from stem import process
from stem.control import Controller
if not os.path.exists('onionr.sh'):
os.chdir('../')
sys.path.append("src/")
from base64 import b32encode
from hashlib import sha3_256
try:
sys.argv[1]
amount = int(sys.argv[1])
except IndexError:
sys.exit(1)
amount = 1
tor_process = process.launch_tor_with_config(
completion_percent=0,
config = {
'ControlPort': '2778',
'DisableNetwork': '1',
'Log': [
'NOTICE stdout',
'ERR file /tmp/tor_error_log',
],
},
)
with Controller.from_port('127.0.0.1', 2778) as controller:
controller.authenticate()
for i in range(1024, 1024 + int(sys.argv[1])):
hs = controller.create_ephemeral_hidden_service(
{80: i},
key_type='NEW',
key_content='ED25519-V3',
await_publication=False,
detached=True)
print(hs.service_id + ".onion")
controller.remove_ephemeral_hidden_service(hs.service_id)
tor_process.kill()
version = int(3).to_bytes(1, "little")
for i in range(amount):
pubkey = os.urandom(32)
#digest = sha3_256(b".onion checksum" + pubkey + version).digest()[:2]
digest = sha3_256()
digest.update(b".onion checksum")
digest.update(pubkey)
digest.update(version)
digest = digest.digest()[:2]
print(b32encode(pubkey + digest + version).decode().lower() + ".onion")

View File

@ -6,7 +6,7 @@ if not os.path.exists('onionr.sh'):
os.chdir('../')
sys.path.append("src/")
from coredb.blockmetadb import get_block_list
from onionrblocks.onionrblockapi import Block
from oldblocks.onionrblockapi import Block
for bl in get_block_list():
bl_obj = Block(bl, decrypt=False)

View File

@ -5,8 +5,8 @@ import os
if not os.path.exists('onionr.sh'):
os.chdir('../')
sys.path.append("src/")
import onionrblocks
import oldblocks
expire = 600
print(onionrblocks.insert(data=os.urandom(32), expire=expire))
print(oldblocks.insert(data=os.urandom(32), expire=expire))

View File

@ -70,7 +70,6 @@ createdirs.create_dirs()
import bigbrother # noqa
from onionrcommands import parser # noqa
from onionrplugins import onionrevents as events # noqa
from onionrblocks.deleteplaintext import delete_plaintext_no_blacklist # noqa
setup.setup_config()
@ -84,8 +83,6 @@ if config.get('advanced.security_auditing', True):
except onionrexceptions.PythonVersion:
pass
if not config.get('general.store_plaintext_blocks', True):
delete_plaintext_no_blacklist()
setup.setup_default_plugins()

24
src/anonvdf-block-creator.py Executable file
View File

@ -0,0 +1,24 @@
#!/usr/bin/env python
from base64 import b85decode
import sys
import os
import ujson as json
from onionrblocks import blockcreator
# This script creates a block without storing it. it is written to stdout
# It is used instead of in the main process to avoid GIL locking/slow down
metadata = json.loads(sys.argv[1])
block_type = sys.argv[2]
ttl = int(sys.argv[3])
data = b85decode(sys.stdin.read())
with os.fdopen(sys.stdout.fileno(), 'wb') as stdout:
bl = blockcreator.create_anonvdf_block(data, block_type, ttl, **metadata)
try:
stdout.write(bl.id + bl.get_packed())
except BrokenPipeError:
pass

38
src/anonvdf-block-validator.py Executable file
View File

@ -0,0 +1,38 @@
#!/usr/bin/env python
# This is a subprocess because block validation is somewhat CPU intensive
from base64 import b85decode, b85encode
import os
from sys import argv, stdin, stderr, stdout, exit
from kasten import Kasten
from kasten.exceptions import InvalidID
from onionrblocks.exceptions import BlockExpired
from onionrblocks.generators import AnonVDFGenerator
block_hash = b85decode(argv[1])
block_bytes = b85decode(stdin.read())
try:
Kasten(
block_hash, block_bytes,
AnonVDFGenerator, auto_check_generator=True)
except InvalidID:
stderr.write(
"Invalid block ID for " +
b85encode(block_hash).decode('utf-8'))
except ValueError as e:
# Supposed to be if rounds are not specified in the block
stderr.write(e.message)
except BlockExpired:
stderr.write(
b85encode(block_hash).decode('utf-8') + " is expired")
else:
with os.fdopen(stdout.fileno(), 'wb') as std:
std.write(b"valid")
exit(0)
stderr.flush()
exit(1)

View File

@ -6,4 +6,3 @@ Contains the WSGI servers Onionr uses for remote peer communication and local da
* \_\_init\_\_.py: Exposes the server classes
* private: Contains the client API (the server used to interact with the local Onionr daemon, and view the web UI)
* public: Contains the public API (the server used by remote peers to talk to our daemon)

View File

@ -4,7 +4,6 @@ Public is net-facing server meant for other nodes
Private is meant for controlling and accessing this node
"""
from . import public, private
from . import private
PublicAPI = public.PublicAPI
ClientAPI = private.PrivateAPI

View File

@ -2,6 +2,7 @@
This file handles all incoming http requests to the client, using Flask
"""
import http
from typing import Dict
import hmac
@ -16,20 +17,19 @@ import logger
from etc import waitforsetvar
from . import register_private_blueprints
import config
from .. import public
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
@ -50,7 +50,7 @@ class PrivateAPI:
self.startTime = epoch.get_epoch()
app = flask.Flask(__name__)
bind_port = int(config.get('client.client.port', 59496))
self.bindPort = bind_port
@ -68,7 +68,7 @@ class PrivateAPI:
self.httpServer = ''
self.queueResponse = {}
self.get_block_data = httpapi.apiutils.GetBlockData(self)
register_private_blueprints.register_private_blueprints(self, app)
httpapi.load_plugin_blueprints(app)
self.app = app
@ -77,17 +77,12 @@ class PrivateAPI:
"""Start client gevent API web server with flask client app."""
waitforsetvar.wait_for_set_var(self, "_too_many")
fd_handler = httpapi.fdsafehandler.FDSafeHandler
self.publicAPI = self._too_many.get( # pylint: disable=E1101
public.PublicAPI)
self._too_many.add(httpapi.wrappedfunctions.SubProcVDFGenerator(self._too_many))
self.httpServer = WSGIServer((self.host, self.bindPort),
self.app, log=None,
handler_class=fd_handler)
self.httpServer.serve_forever()
def setPublicAPIInstance(self, inst):
"""Dynamically set public API instance."""
self.publicAPI = inst
def validateToken(self, token):
"""Validate that the client token matches the given token.
@ -110,10 +105,3 @@ class PrivateAPI:
# Don't error on race condition with startup
pass
def getBlockData(self, bHash, decrypt=False, raw=False,
headerOnly=False) -> bytes:
"""Returns block data bytes."""
return self.get_block_data.get_block_data(bHash,
decrypt=decrypt,
raw=raw,
headerOnly=headerOnly)

View File

@ -5,25 +5,25 @@ This file registers blueprints for the private api server
from threading import Thread
from gevent import sleep
from httpapi import security, friendsapi, configapi, insertblock
from httpapi import miscclientapi, onionrsitesapi, apiutils
from httpapi import security, configapi
from httpapi import miscclientapi, apiutils
from httpapi import themeapi
from httpapi import fileoffsetreader
from httpapi.sse.private import private_sse_blueprint
from httpapi.serializedapi import serialized_api_bp
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
@ -31,19 +31,15 @@ def register_private_blueprints(private_api, app):
"""Register private API plask blueprints."""
app.register_blueprint(security.client.ClientAPISecurity(
private_api).client_api_security_bp)
app.register_blueprint(friendsapi.friends)
app.register_blueprint(configapi.config_BP)
app.register_blueprint(insertblock.ib)
app.register_blueprint(miscclientapi.getblocks.client_get_blocks)
app.register_blueprint(miscclientapi.endpoints.PrivateEndpoints(
private_api).private_endpoints_bp)
app.register_blueprint(miscclientapi.motd.bp)
app.register_blueprint(onionrsitesapi.site_api)
app.register_blueprint(apiutils.shutdown.shutdown_bp)
app.register_blueprint(miscclientapi.staticfiles.static_files_bp)
app.register_blueprint(themeapi.theme_blueprint)
app.register_blueprint(private_sse_blueprint)
app.register_blueprint(fileoffsetreader.offset_reader_api)
app.register_blueprint(serialized_api_bp)
def _add_events_bp():
while True:
@ -55,6 +51,6 @@ def register_private_blueprints(private_api, app):
app.register_blueprint(
private_api._too_many.get_by_string('DaemonEventsBP').flask_bp)
Thread(target=_add_events_bp).start()
Thread(target=_add_events_bp, name='Private blueprints adder').start()
return app

View File

@ -1,76 +0,0 @@
"""Onionr - Private P2P Communication.
This file handles all incoming http requests
to the public api server, using Flask
"""
import time
import threading
import flask
from gevent.pywsgi import WSGIServer
from httpapi import apiutils, security, fdsafehandler, miscpublicapi
import logger
import config
import filepaths
from utils import gettransports
from etc import onionrvalues, waitforsetvar
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def _get_tor_adder(pub_api):
transports = []
while len(transports) == 0:
transports = gettransports.get()
time.sleep(0.3)
pub_api.torAdder = transports[0]
class PublicAPI:
"""The new client api server, isolated from the public api."""
def __init__(self):
"""Setup the public api app."""
app = flask.Flask('PublicAPI')
app.config['MAX_CONTENT_LENGTH'] = 5 * 1024 * 1024
self.i2pEnabled = config.get('i2p.host', False)
self.hideBlocks = [] # Blocks to be denied sharing
self.host = apiutils.setbindip.set_bind_IP(
filepaths.public_API_host_file)
threading.Thread(target=_get_tor_adder,
args=[self], daemon=True).start()
self.torAdder = ""
self.bindPort = config.get('client.public.port')
self.lastRequest = 0
# total rec requests to public api since server started
self.hitCount = 0
self.config = config
self.API_VERSION = onionrvalues.API_VERSION
logger.info('Running public api on %s:%s' % (self.host, self.bindPort))
app.register_blueprint(
security.public.PublicAPISecurity(self).public_api_security_bp)
app.register_blueprint(
miscpublicapi.endpoints.PublicEndpoints(self).public_endpoints_bp)
self.app = app
def start(self):
"""Start the Public API server."""
waitforsetvar.wait_for_set_var(self, "_too_many")
self.httpServer = WSGIServer((self.host, self.bindPort),
self.app, log=None,
handler_class=fdsafehandler.FDSafeHandler)
self.httpServer.serve_forever()

View File

@ -35,6 +35,9 @@ def detect_socket_leaks(socket_event):
try:
ip_address = ipaddress.ip_address(ip_address)
except ValueError:
if ip_address == "/":
# unix socket
return
logger.warn(f'Conn made to {ip_address} outside of Tor/similar')
raise \
NetworkLeak('Conn to host/non local IP, this is a privacy issue!')

View File

@ -3,11 +3,12 @@
Prevent eval/exec/os.system and log it
"""
import base64
import platform
from os import read
import logger
from utils import identifyhome
from onionrexceptions import ArbitraryCodeExec
from utils import readstatic
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -36,6 +37,7 @@ def block_exec(event, info):
# because libraries have stupid amounts of compile/exec/eval,
# We have to use a whitelist where it can be tolerated
# Generally better than nothing, not a silver bullet
return
whitelisted_code = [
'netrc.py',
'shlex.py',
@ -56,14 +58,30 @@ def block_exec(event, info):
'stem/response/mapaddress.py',
'stem/response/protocolinfo.py',
'apport/__init__.py',
'apport/report.py'
'apport/report.py',
'gevent/pool.py',
'gevent/queue.py',
'gevent/lock.py',
'gevent/monkey.py',
'gevent/_semaphore.py',
'gevent/_imap.py'
]
whitelisted_source = []
home = identifyhome.identify_home()
try:
whitelisted_source = readstatic.read_static(
'base64-code-whitelist.txt')
whitelisted_source = whitelisted_source.splitlines()
except FileNotFoundError:
logger.warn("Failed to read whitelisted code for bigbrother")
whitelisted_source = []
code_b64 = base64.b64encode(info[0].co_code).decode()
if code_b64 in whitelisted_source:
return
#uncomment when you want to build on the whitelist
else:
with open("../static-data/base64-code-whitelist.txt", "a") as f:
f.write(code_b64 + "\n")
return
for source in whitelisted_code:
if info[0].co_filename.endswith(source):

View File

@ -0,0 +1,83 @@
"""Onionr - Private P2P Communication.
BlockCreatorQueue, generate anonvdf blocks in a queue
"""
from typing import Callable
from threading import Thread
from hashlib import sha3_224
from os import cpu_count
import time
import blockio
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
class AlreadyGenerating(Exception): pass # noqa
class PassToSafeDB:
def __init__(self, db: 'SafeDB'):
self.db = db
self.block_creator_queue = BlockCreatorQueue(self.store_kasten)
def store_kasten(self, kasten_object):
self.db.put(kasten_object.id, kasten_object.get_packed())
def queue_then_store(self, block_data, block_type, ttl, **block_metadata):
self.block_creator_queue.queue_block(block_data, block_type, ttl, **block_metadata)
class BlockCreatorQueue:
def __init__(
self, callback_func: Callable, *additional_callback_func_args,
**additional_callback_func_kwargs):
self.callback_func = callback_func
self.queued = set()
self.max_parallel = cpu_count()
self.additional_callback_func_args = additional_callback_func_args
self.additional_callback_func_kwargs = additional_callback_func_kwargs
def block_data_in_queue(self, block_data: bytes) -> bool:
if sha3_224(block_data).digest() in self.queued:
return True
return False
def queue_block(
self, block_data, block_type, ttl: int, **block_metadata) -> bytes:
"""Spawn a thread to make a subprocess to generate a block
if queue is not full, else wait"""
digest = sha3_224(block_data).digest()
def _do_create():
if digest in self.queued:
raise AlreadyGenerating()
self.queued.add(digest)
while len(self.queued) >= self.max_parallel:
time.sleep(1)
result = blockio.subprocgenerate.vdf_block(
block_data, block_type, ttl, **block_metadata)
self.queued.remove(digest)
self.callback_func(
result,
*self.additional_callback_func_args,
**self.additional_callback_func_kwargs)
Thread(
target=_do_create, daemon=True,
name="BlockCreatorQueue block creation").start()
return digest

23
src/blockio/__init__.py Normal file
View File

@ -0,0 +1,23 @@
"""Onionr - Private P2P Communication.
Wrap safedb for storing and fetching blocks
"""
from .store import store_block
from .load import load_block, list_blocks_by_type, list_all_blocks
from .clean import clean_expired_blocks, clean_block_list_entries
from . import subprocgenerate
from . import subprocvalidate
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""

View File

@ -0,0 +1,2 @@
from .cleanexpired import clean_expired_blocks
from .cleanblocklistentries import clean_block_list_entries

View File

@ -0,0 +1,33 @@
"""Onionr - Private P2P Communication.
Delete block type lists that are empty
"""
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from safedb import SafeDB
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def clean_block_list_entries(db: 'SafeDB'):
key = db.db_conn.firstkey()
delete_keys = []
while key:
if key.startswith(b'bl-'):
if not db.get(key):
delete_keys.append(key)
key = db.db_conn.nextkey(key)
for key in delete_keys:
del db.db_conn[key]

View File

@ -0,0 +1,48 @@
"""Onionr - Private P2P Communication.
clean expired blocks
"""
from typing import TYPE_CHECKING
from kasten import Kasten
from onionrblocks.generators.anonvdf import AnonVDFGenerator
from onionrblocks.exceptions import BlockExpired
if TYPE_CHECKING:
from safedb import SafeDB
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def clean_expired_blocks(db: 'SafeDB'):
key = db.db_conn.firstkey()
delete_list = set()
# Scan all database keys and check kasten objs if they are a hash
while key:
try:
if key.startswith(b'bl-') or key.startswith(b'enc'):
key = db.db_conn.nextkey(key)
continue
Kasten(key, db.get(key), AnonVDFGenerator)
except BlockExpired:
block_type = Kasten(
key, db.get(key),
None, auto_check_generator=False).get_data_type()
db.db_conn[f'bl-{block_type}'] = \
db.db_conn[f'bl-{block_type}'].replace(key, b'')
delete_list.add(key)
key = db.db_conn.nextkey(key)
for key in delete_list:
del db.db_conn[key]

View File

@ -0,0 +1,54 @@
"""Onionr - Private P2P Communication.
Get blocks from safedb
"""
from typing import TYPE_CHECKING, List
from kasten import Kasten
if TYPE_CHECKING:
from kasten.types import BlockChecksumBytes
from safedb import SafeDB
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def load_block(block: 'BlockChecksumBytes', safe_db: 'SafeDB') -> Kasten:
return Kasten(
block, safe_db.get(block), generator=None, auto_check_generator=False)
def list_blocks_by_type(
block_type: str, safe_db: 'SafeDB') -> List['BlockChecksumBytes']:
try:
block_type = block_type.decode('utf-8')
except AttributeError:
pass
blocks = safe_db.get(f'bl-{block_type}')
return zip(*[iter(blocks)]*64)
def list_all_blocks(safe_db: 'SafeDB'):
# Builds and return a master list of blocks by identifying all type lists
# and iterating them
key = safe_db.db_conn.firstkey()
master_list = []
while key:
if key.startswith(b'bl-'):
master_list.extend(
list(list_blocks_by_type(key.replace(b'bl-', b''), safe_db)))
key = safe_db.db_conn.nextkey(key)
return master_list

View File

@ -0,0 +1,43 @@
"""Onionr - Private P2P Communication.
Store blocks and cache meta info such as block type
"""
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from kasten import Kasten
from safedb import SafeDB
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def store_block(block: 'Kasten', safe_db: 'SafeDB', own_block=False):
# This does not handle validation of blocks
# safe_db is initialized by the daemon when starting normally
# so any other commands need to initialize it seperately
block_type = block.get_data_type()
try:
block_list_for_type = safe_db.get(f'bl-{block_type}')
if block.id in block_list_for_type:
raise ValueError("Cannot store duplicate block")
except KeyError:
block_list_for_type = b''
safe_db.put(block.id, block.get_packed())
# Append the block to the list of blocks for this given type
block_list_for_type += block.id
safe_db.put(f'bl-{block_type}', block_list_for_type)

View File

@ -0,0 +1,45 @@
from base64 import b85encode
import os
import subprocess
import threading
import ujson as json
import kasten
from onionrplugins import onionrevents
from blockio import store_block
from onionrblocks.generators.anonvdf import AnonVDFGenerator
_DIR = os.path.dirname(os.path.realpath(__file__)) + '/../'
def vdf_block(data, data_type, ttl, **metadata):
try:
data = data.encode('utf-8')
except AttributeError:
pass
data = b85encode(data)
generated = subprocess.Popen(
[
f'{_DIR}anonvdf-block-creator.py',
json.dumps(metadata),
data_type,
str(ttl)],
stdout=subprocess.PIPE,
stdin=subprocess.PIPE)
generated = generated.communicate(data)[0]
return kasten.Kasten(
generated[:64], generated[64:],
AnonVDFGenerator, auto_check_generator=True)
def gen_and_store_vdf_block(shared_state, *args, **kwargs):
safe_db = shared_state.get_by_string('SafeDB')
k = vdf_block(*args, **kwargs)
store_block(
k,
safe_db,
own_block=True
)
onionrevents.event('blockcreated', data=shared_state, threaded=True)

View File

@ -0,0 +1,19 @@
import subprocess
import os
from base64 import b85encode
from onionrblocks.generators import anonvdf
_DIR = os.path.dirname(os.path.realpath(__file__)) + '/../'
def vdf_block(block_hash: bytes, block_data: bytes):
block_data = b85encode(block_data)
p = subprocess.Popen(
[
f'{_DIR}anonvdf-block-validator.py',
b85encode(block_hash)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
result = p.communicate(block_data)
if result[1]:
raise anonvdf.InvalidID()

View File

@ -8,37 +8,22 @@ import time
import config
import logger
import onionrpeers
import onionrplugins as plugins
from . import onlinepeers
from . import uploadqueue
from communicatorutils import downloadblocks
from communicatorutils import lookupblocks
from communicatorutils import lookupadders
from communicatorutils import connectnewpeers
from communicatorutils import uploadblocks
from communicatorutils import announcenode, deniableinserts
from communicatorutils import cooldownpeer
from communicatorutils import housekeeping
from communicatorutils import netcheck
from onionrthreads import add_onionr_thread
from onionrcommands.openwebinterface import get_url
from netcontroller import NetController
from . import bootstrappeers
from . import daemoneventhooks
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
config.reload()
@ -61,11 +46,7 @@ class OnionrCommunicatorDaemon:
if config.get('general.offline_mode', False):
self.kv.put('isOnline', False)
# initialize core with Tor socks port being 3rd argument
self.proxyPort = shared_state.get(NetController).socksPort
self.upload_session_manager = self.shared_state.get(
uploadblocks.sessionmanager.BlockUploadSessionManager)
self.shared_state.share_object()
# loop time.sleep delay in seconds
@ -77,67 +58,6 @@ class OnionrCommunicatorDaemon:
# Loads in and starts the enabled plugins
plugins.reload()
# extends our upload list and saves our list when Onionr exits
uploadqueue.UploadQueue(self)
add_onionr_thread(
lookupblocks.lookup_blocks_from_communicator,
[self.shared_state], 25, 3)
add_onionr_thread(
downloadblocks.download_blocks_from_communicator,
[self.shared_state],
config.get('timers.getBlocks', 10), 1)
add_onionr_thread(onlinepeers.clear_offline_peer, [self.kv], 58)
add_onionr_thread(
housekeeping.clean_old_blocks, [self.shared_state], 10, 1)
# Discover new peers
add_onionr_thread(
lookupadders.lookup_new_peer_transports_with_communicator,
[shared_state], 60, 3)
# Timer for adjusting which peers
# we actively communicate to at any given time,
# to avoid over-using peers
add_onionr_thread(
cooldownpeer.cooldown_peer, [self.shared_state], 30, 60)
# Timer to read the upload queue and upload the entries to peers
add_onionr_thread(
uploadblocks.upload_blocks_from_communicator,
[self.shared_state], 5, 1)
# This timer creates deniable blocks,
# in an attempt to further obfuscate block insertion metadata
if config.get('general.insert_deniable_blocks', True):
add_onionr_thread(
deniableinserts.insert_deniable_block, [], 180, 10)
if config.get('transports.tor', True):
# Timer to check for connectivity,
# through Tor to various high-profile onion services
add_onionr_thread(netcheck.net_check, [shared_state], 500, 60)
# Announce the public API server transport address
# to other nodes if security level allows
if config.get('general.security_level', 1) == 0 \
and config.get('general.announce_node', True):
# Default to high security level incase config breaks
add_onionr_thread(
announcenode.announce_node, [self.shared_state], 600, 60)
else:
logger.debug('Will not announce node.')
add_onionr_thread(onionrpeers.peer_cleanup, [], 300, 300)
add_onionr_thread(housekeeping.clean_keys, [], 15, 1)
if config.get('general.use_bootstrap_list', True):
bootstrappeers.add_bootstrap_list_to_peer_list(
self.kv, [], db_only=True)
daemoneventhooks.daemon_event_handlers(shared_state)
@ -169,38 +89,6 @@ class OnionrCommunicatorDaemon:
logger.info(
'Goodbye. (Onionr is cleaning up, and will exit)', terminal=True)
def decrementThreadCount(self, threadName):
"""Decrement amount of a thread name if more than zero.
called when a function meant to be run in a thread ends
"""
try:
if self.threadCounts[threadName] > 0:
self.threadCounts[threadName] -= 1
except KeyError:
pass
def peerCleanup(self):
"""This just calls onionrpeers.cleanupPeers.
Remove dead or bad peers (offline too long, too slow)"""
onionrpeers.peer_cleanup()
self.decrementThreadCount('peerCleanup')
def getPeerProfileInstance(self, peer):
"""Gets a peer profile instance from the list of profiles"""
for i in self.kv.get('peerProfiles'):
# if the peer's profile is already loaded, return that
if i.address == peer:
retData = i
break
else:
# if the peer's profile is not loaded, return a new one.
# connectNewPeer also adds it to the list on connect
retData = onionrpeers.PeerProfiles(peer)
self.kv.get('peerProfiles').append(retData)
return retData
def startCommunicator(shared_state):
OnionrCommunicatorDaemon(shared_state)

View File

@ -1,36 +0,0 @@
"""Onionr - Private P2P Communication.
add bootstrap peers to the communicator peer list
"""
from typing import TYPE_CHECKING
from utils import readstatic, gettransports
from coredb import keydb
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
bootstrap_peers = readstatic.read_static('bootstrap-nodes.txt').split(',')
def add_bootstrap_list_to_peer_list(kv, peerList, db_only=False):
"""Add the bootstrap list to the peer list (no duplicates)."""
for i in bootstrap_peers:
# Add bootstrap peers to peerList (does not save them)
# Don't add them if they're already added or in the offline list
if i not in peerList and i not in kv.get('offlinePeers') \
and i not in gettransports.get() and len(str(i).strip()) > 0:
if not db_only:
peerList.append(i)
keydb.addkeys.add_address(i)

View File

@ -4,14 +4,10 @@ Hooks to handle daemon events
"""
from threading import Thread
from .removefrominsertqueue import remove_from_insert_queue
from typing import TYPE_CHECKING
from gevent import sleep
from communicatorutils.uploadblocks import mixmate
from communicatorutils import restarttor
if TYPE_CHECKING:
from toomanyobjs import TooMany
@ -19,7 +15,6 @@ if TYPE_CHECKING:
from communicator import OnionrCommunicatorDaemon
from httpapi.daemoneventsapi import DaemonEventsBP
from onionrtypes import BlockHash
from apiservers import PublicAPI
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -44,39 +39,16 @@ def daemon_event_handlers(shared_state: 'TooMany'):
except KeyError:
sleep(0.2)
comm_inst = _get_inst('OnionrCommunicatorDaemon')
public_api: 'PublicAPI' = _get_inst('PublicAPI')
events_api: 'DaemonEventsBP' = _get_inst('DaemonEventsBP')
kv: 'DeadSimpleKV' = _get_inst('DeadSimpleKV')
def remove_from_insert_queue_wrapper(block_hash: 'BlockHash'):
remove_from_insert_queue(comm_inst, block_hash)
return "removed"
def print_test(text=''):
print("It works!", text)
return f"It works! {text}"
def upload_event(block: 'BlockHash' = ''):
if not block:
raise ValueError
public_api.hideBlocks.append(block)
try:
mixmate.block_mixer(kv.get('blocksToUpload'), block)
except ValueError:
pass
return "removed"
def restart_tor():
restarttor.restart(shared_state)
kv.put('offlinePeers', [])
kv.put('onlinePeers', [])
def test_runtime():
Thread(target=comm_inst.shared_state.get_by_string(
"OnionrRunTestManager").run_tests).start()
events_api.register_listener(remove_from_insert_queue_wrapper)
events_api.register_listener(restart_tor)
events_api.register_listener(print_test)
events_api.register_listener(upload_event)
events_api.register_listener(test_runtime)

View File

@ -1,33 +0,0 @@
"""Onionr - P2P Anonymous Storage Network.
Remove block hash from daemon's upload list.
"""
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
from communicator import OnionrCommunicatorDaemon
from onionrtypes import BlockHash
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def remove_from_insert_queue(comm_inst: "OnionrCommunicatorDaemon",
b_hash: "BlockHash"):
"""Remove block hash from daemon's upload list."""
kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string("DeadSimpleKV")
try:
kv.get('generating_blocks').remove(b_hash)
except ValueError:
pass

View File

@ -1,12 +0,0 @@
# Online Peers
Manages a pool of peers to perform actions with. Since Onionr does not maintain socket connections, it holds a list of peers.
## Files
* \_\_init\_\_.py: exposes some functions to interact with the pool
* clearofflinepeer.py: Pop the oldest peer in the offline list
* onlinepeers.py: communicator timer to add new peers to the pool randomly
* pickonlinepeers.py: returns a random peer from the online pool
* removeonlinepeer.py: removes a specified peer from the online pool

View File

@ -1,6 +0,0 @@
from . import clearofflinepeer, onlinepeers, pickonlinepeers, removeonlinepeer
clear_offline_peer = clearofflinepeer.clear_offline_peer
get_online_peers = onlinepeers.get_online_peers
pick_online_peer = pickonlinepeers.pick_online_peer
remove_online_peer = removeonlinepeer.remove_online_peer

View File

@ -1,35 +0,0 @@
"""Onionr - Private P2P Communication.
clear offline peer in a communicator instance
"""
from typing import TYPE_CHECKING
import logger
if TYPE_CHECKING:
from communicator import OnionrCommunicatorDaemon
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def clear_offline_peer(kv: 'DeadSimpleKV'):
"""Remove the longest offline peer to retry later."""
try:
removed = kv.get('offlinePeers').pop(0)
except IndexError:
pass
else:
logger.debug('Removed ' + removed +
' from offline list, will try them again.')

View File

@ -1,63 +0,0 @@
"""Onionr - Private P2P Communication.
get online peers in a communicator instance
"""
import time
from typing import TYPE_CHECKING
import config
from etc.humanreadabletime import human_readable_time
from communicatorutils.connectnewpeers import connect_new_peer_to_communicator
import logger
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def get_online_peers(shared_state):
"""Manage the kv.get('onlinePeers') attribute list.
Connect to more peers if we have none connected
"""
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
if config.get('general.offline_mode', False):
return
logger.info('Refreshing peer pool...')
max_peers = int(config.get('peers.max_connect', 10))
needed = max_peers - len(kv.get('onlinePeers'))
last_seen = 'never'
if not isinstance(kv.get('lastNodeSeen'), type(None)):
last_seen = human_readable_time(kv.get('lastNodeSeen'))
for _ in range(needed):
if len(kv.get('onlinePeers')) == 0:
connect_new_peer_to_communicator(shared_state, useBootstrap=True)
else:
connect_new_peer_to_communicator(shared_state)
if kv.get('shutdown'):
break
else:
if len(kv.get('onlinePeers')) == 0:
logger.debug('Couldn\'t connect to any peers.' +
f' Last node seen {last_seen} ago.')
try:
get_online_peers(shared_state)
except RecursionError:
pass
else:
kv.put('lastNodeSeen', time.time())

View File

@ -1,47 +0,0 @@
"""
Onionr - Private P2P Communication.
pick online peers in a communicator instance
"""
import secrets
from typing import TYPE_CHECKING
import onionrexceptions
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def pick_online_peer(kv: 'DeadSimpleKV'):
"""Randomly picks peer from pool without bias (using secrets module)."""
ret_data = ''
peer_length = len(kv.get('onlinePeers'))
if peer_length <= 0:
raise onionrexceptions.OnlinePeerNeeded
while True:
peer_length = len(kv.get('onlinePeers'))
try:
# Get a random online peer, securely.
# May get stuck in loop if network is lost
ret_data = kv.get('onlinePeers')[secrets.randbelow(peer_length)]
except IndexError:
pass
else:
break
return ret_data

View File

@ -1,38 +0,0 @@
"""Onionr - Private P2P Communication.
remove an online peer from the pool in a communicator instance
"""
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def remove_online_peer(kv, peer):
"""Remove an online peer."""
try:
del kv.get('connectTimes')[peer]
except KeyError:
pass
try:
del kv.get('dbTimestamps')[peer]
except KeyError:
pass
try:
kv.get('onlinePeers').remove(peer)
except ValueError:
pass

View File

@ -1,78 +0,0 @@
"""Onionr - Private P2P Communication.
This file implements logic for performing requests to Onionr peers
"""
from typing import TYPE_CHECKING
import streamedrequests
import logger
from onionrutils import epoch, basicrequests
from coredb import keydb
from . import onlinepeers
from onionrtypes import OnionAddressString
from onionrpeers.peerprofiles import PeerProfiles
from etc.waitforsetvar import wait_for_set_var
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def get_peer_profile(kv, address: OnionAddressString) -> 'PeerProfiles':
profile_inst_list = kv.get('peerProfiles')
for profile in profile_inst_list:
if profile.address == address:
return profile
p = PeerProfiles(address)
profile_inst_list.append(p)
return p
def peer_action(shared_state, peer, action,
returnHeaders=False, max_resp_size=5242880):
"""Perform a get request to a peer."""
penalty_score = -10
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
if len(peer) == 0:
return False
url = 'http://%s/%s' % (peer, action)
try:
ret_data = basicrequests.do_get_request(url, port=kv.get('proxyPort'),
max_size=max_resp_size)
except streamedrequests.exceptions.ResponseLimitReached:
logger.warn(
'Request failed due to max response size being overflowed',
terminal=True)
ret_data = False
penalty_score = -100
# if request failed, (error), mark peer offline
if ret_data is False:
try:
get_peer_profile(kv, peer).addScore(penalty_score)
onlinepeers.remove_online_peer(kv, peer)
keydb.transportinfo.set_address_info(
peer, 'lastConnectAttempt', epoch.get_epoch())
if action != 'ping' and not kv.get('shutdown'):
logger.warn(f'Lost connection to {peer}', terminal=True)
# Will only add a new peer to pool if needed
onlinepeers.get_online_peers(kv)
except ValueError:
pass
else:
peer_profile = get_peer_profile(kv, peer)
peer_profile.update_connect_time()
peer_profile.addScore(1)
# If returnHeaders, returns tuple of data, headers.
# If not, just data string
return ret_data

View File

@ -1,73 +0,0 @@
"""Onionr - Private P2P Communication.
Class to remember blocks that need to be uploaded
and not shared on startup/shutdown
"""
import atexit
import os
from typing import TYPE_CHECKING
import deadsimplekv
import filepaths
from onionrutils import localcommand
if TYPE_CHECKING:
from communicator import OnionrCommunicatorDaemon
from deadsimplekv import DeadSimpleKV
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
UPLOAD_MEMORY_FILE = filepaths.upload_list
def _add_to_hidden_blocks(cache):
for bl in cache:
localcommand.local_command('waitforshare/' + bl, post=True)
class UploadQueue:
"""Saves and loads block upload info from json file."""
def __init__(self, communicator: 'OnionrCommunicatorDaemon'):
"""Start the UploadQueue object, loading left over uploads into queue.
register save shutdown function
"""
self.communicator = communicator
cache: deadsimplekv.DeadSimpleKV = deadsimplekv.DeadSimpleKV(
UPLOAD_MEMORY_FILE)
self.kv: "DeadSimpleKV" = \
communicator.shared_state.get_by_string("DeadSimpleKV")
self.store_obj = cache
cache = cache.get('uploads')
if cache is None:
cache = []
_add_to_hidden_blocks(cache)
self.kv.get('blocksToUpload').extend(cache)
atexit.register(self.save)
def save(self):
"""Save to disk on shutdown or if called manually."""
bl: deadsimplekv.DeadSimpleKV = self.kv.get('blocksToUpload')
if len(bl) == 0:
try:
os.remove(UPLOAD_MEMORY_FILE)
except FileNotFoundError:
pass
else:
self.store_obj.put('uploads', bl)
self.store_obj.flush()

View File

@ -1,33 +0,0 @@
# communicatorutils
The files in this submodule handle various subtasks and utilities for the onionr communicator.
## Files:
announcenode.py: Uses a communicator instance to announce our transport address to connected nodes
connectnewpeers.py: takes a communicator instance and has it connect to as many peers as needed, and/or to a new specified peer.
cooldownpeer.py: randomly selects a connected peer in a communicator and disconnects them for the purpose of security and network balancing.
daemonqueuehandler.py: checks for new commands in the daemon queue and processes them accordingly.
deniableinserts.py: insert fake blocks with the communicator for plausible deniability
downloadblocks.py: iterates a communicator instance's block download queue and attempts to download the blocks from online peers
housekeeping.py: cleans old blocks and forward secrecy keys
lookupadders.py: ask connected peers to share their list of peer transport addresses
lookupblocks.py: lookup new blocks from connected peers from the communicator
netcheck.py: check if the node is online based on communicator status and onion server ping results
onionrcommunicataortimers.py: create a timer for a function to be launched on an interval. Control how many possible instances of a timer may be running a function at once and control if the timer should be ran in a thread or not.
proxypicker.py: returns a string name for the appropriate proxy to be used with a particular peer transport address.
servicecreator.py: iterate connection blocks and create new direct connection servers for them.
uploadblocks.py: iterate a communicator's upload queue and upload the blocks to connected peers

View File

@ -1,77 +0,0 @@
"""
Onionr - Private P2P Communication.
Use a communicator instance to announce
our transport address to connected nodes
"""
from typing import TYPE_CHECKING
import logger
from onionrutils import basicrequests
from utils import gettransports
from netcontroller import NetController
from communicator import onlinepeers
from coredb import keydb
import onionrexceptions
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def announce_node(shared_state):
"""Announce our node to our peers."""
ret_data = False
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
config = shared_state.get_by_string("OnionrCommunicatorDaemon").config
# Do not let announceCache get too large
if len(kv.get('announceCache')) >= 10000:
kv.get('announceCache').popitem()
if config.get('general.security_level', 0) == 0:
# Announce to random online peers
for i in kv.get('onlinePeers'):
if i not in kv.get('announceCache'):
peer = i
break
else:
try:
peer = onlinepeers.pick_online_peer(kv)
except onionrexceptions.OnlinePeerNeeded:
peer = ""
try:
ourID = gettransports.get()[0]
if not peer:
raise onionrexceptions.OnlinePeerNeeded
except (IndexError, onionrexceptions.OnlinePeerNeeded):
pass
else:
url = 'http://' + peer + '/announce'
data = {'node': ourID}
logger.info('Announcing node to ' + url)
if basicrequests.do_post_request(
url,
data,
port=shared_state.get(NetController).socksPort)\
== 'Success':
logger.info('Successfully introduced node to ' + peer,
terminal=True)
ret_data = True
keydb.transportinfo.set_address_info(peer, 'introduced', 1)
return ret_data

View File

@ -1,117 +0,0 @@
"""Onionr - Private P2P Communication.
Connect a new peer to our communicator instance.
Does so randomly if no peer is specified
"""
import time
import secrets
import onionrexceptions
import logger
import onionrpeers
from utils import networkmerger, gettransports
from onionrutils import stringvalidators, epoch
from communicator import peeraction, bootstrappeers
from coredb import keydb
import config
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def connect_new_peer_to_communicator(shared_state, peer='', useBootstrap=False):
retData = False
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
tried = kv.get('offlinePeers')
transports = gettransports.get()
if peer != '':
if stringvalidators.validate_transport(peer):
peerList = [peer]
else:
raise onionrexceptions.InvalidAddress(
'Will not attempt connection test to invalid address')
else:
peerList = keydb.listkeys.list_adders()
mainPeerList = keydb.listkeys.list_adders()
if not peerList:
peerList = onionrpeers.get_score_sorted_peer_list()
"""
If we don't have enough peers connected or random chance,
select new peers to try
"""
if len(peerList) < 8 or secrets.randbelow(4) == 3:
tryingNew = []
for x in kv.get('newPeers'):
if x not in peerList:
peerList.append(x)
tryingNew.append(x)
for i in tryingNew:
kv.get('newPeers').remove(i)
if len(peerList) == 0 or useBootstrap:
# Avoid duplicating bootstrap addresses in peerList
if config.get('general.use_bootstrap_list', True):
bootstrappeers.add_bootstrap_list_to_peer_list(kv, peerList)
for address in peerList:
address = address.strip()
# Don't connect to our own address
if address in transports:
continue
"""Don't connect to invalid address or
if its already been tried/connected, or if its cooled down
"""
if len(address) == 0 or address in tried \
or address in kv.get('onlinePeers') \
or address in kv.get('cooldownPeer'):
continue
if kv.get('shutdown'):
return
# Ping a peer,
ret = peeraction.peer_action(shared_state, address, 'ping')
if ret == 'pong!':
time.sleep(0.1)
if address not in mainPeerList:
# Add a peer to our list if it isn't already since it connected
networkmerger.mergeAdders(address)
if address not in kv.get('onlinePeers'):
logger.info('Connected to ' + address, terminal=True)
kv.get('onlinePeers').append(address)
kv.get('connectTimes')[address] = epoch.get_epoch()
retData = address
# add peer to profile list if they're not in it
for profile in kv.get('peerProfiles'):
if profile.address == address:
break
else:
kv.get('peerProfiles').append(
onionrpeers.PeerProfiles(address))
try:
del kv.get('plaintextDisabledPeers')[address]
except KeyError:
pass
if peeraction.peer_action(
shared_state, address, 'plaintext') == 'false':
kv.get('plaintextDisabledPeers')[address] = True
break
else:
# Mark a peer as tried if they failed to respond to ping
tried.append(address)
logger.debug('Failed to connect to %s: %s ' % (address, ret))
return retData

View File

@ -1,60 +0,0 @@
"""Onionr - Private P2P Communication.
Select random online peer in a communicator instance and have them "cool down"
"""
from typing import TYPE_CHECKING
from onionrutils import epoch
from communicator import onlinepeers
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def cooldown_peer(shared_state):
"""Randomly add an online peer to cooldown, so we can connect a new one."""
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
config = shared_state.get_by_string("OnionrCommunicatorDaemon").config
online_peer_amount = len(kv.get('onlinePeers'))
minTime = 300
cooldown_time = 600
to_cool = ''
tempConnectTimes = dict(kv.get('connectTimes'))
# Remove peers from cooldown that have been there long enough
tempCooldown = dict(kv.get('cooldownPeer'))
for peer in tempCooldown:
if (epoch.get_epoch() - tempCooldown[peer]) >= cooldown_time:
del kv.get('cooldownPeer')[peer]
# Cool down a peer, if we have max connections alive for long enough
if online_peer_amount >= config.get('peers.max_connect', 10, save=True):
finding = True
while finding:
try:
to_cool = min(tempConnectTimes, key=tempConnectTimes.get)
if (epoch.get_epoch() - tempConnectTimes[to_cool]) < minTime:
del tempConnectTimes[to_cool]
else:
finding = False
except ValueError:
break
else:
onlinepeers.remove_online_peer(kv, to_cool)
kv.get('cooldownPeer')[to_cool] = epoch.get_epoch()

View File

@ -1,35 +0,0 @@
"""Onionr - Private P2P Communication.
Use the communicator to insert fake mail messages
"""
import secrets
from etc import onionrvalues
import onionrblocks
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def insert_deniable_block():
"""Insert a fake block to make it more difficult to track real blocks."""
fakePeer = ''
chance = 10
if secrets.randbelow(chance) == (chance - 1):
# This assumes on the libsodium primitives to have key-privacy
fakePeer = onionrvalues.DENIABLE_PEER_ADDRESS
data = secrets.token_hex(secrets.randbelow(5120) + 1)
onionrblocks.insert(data, header='pm', encryptType='asym',
asymPeer=fakePeer, disableForward=True,
meta={'subject': 'foo'})

View File

@ -1,173 +0,0 @@
"""Onionr - Private P2P Communication.
Download blocks using the communicator instance.
"""
from typing import TYPE_CHECKING
from secrets import SystemRandom
if TYPE_CHECKING:
from communicator import OnionrCommunicatorDaemon
from deadsimplekv import DeadSimpleKV
from gevent import spawn
import onionrexceptions
import logger
import onionrpeers
from communicator import peeraction
from communicator import onlinepeers
from onionrblocks import blockmetadata
from onionrutils import validatemetadata
from coredb import blockmetadb
from onionrutils.localcommand import local_command
import onionrcrypto
import onionrstorage
from onionrblocks import onionrblacklist
from onionrblocks import storagecounter
from . import shoulddownload
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
storage_counter = storagecounter.StorageCounter()
def download_blocks_from_communicator(shared_state: "TooMany"):
"""Use communicator instance to download blocks in the comms's queue"""
blacklist = onionrblacklist.OnionrBlackList()
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
LOG_SKIP_COUNT = 50 # for how many iterations we skip logging the counter
count: int = 0
metadata_validation_result: bool = False
# Iterate the block queue in the communicator
for blockHash in list(kv.get('blockQueue')):
count += 1
try:
blockPeers = list(kv.get('blockQueue')[blockHash])
except KeyError:
blockPeers = []
removeFromQueue = True
if not shoulddownload.should_download(shared_state, blockHash):
continue
if kv.get('shutdown') or not kv.get('isOnline') or \
storage_counter.is_full():
# Exit loop if shutting down or offline, or disk allocation reached
break
# Do not download blocks being downloaded
if blockHash in kv.get('currentDownloading'):
continue
if len(kv.get('onlinePeers')) == 0:
break
# So we can avoid concurrent downloading in other threads of same block
kv.get('currentDownloading').append(blockHash)
if len(blockPeers) == 0:
try:
peerUsed = onlinepeers.pick_online_peer(kv)
except onionrexceptions.OnlinePeerNeeded:
continue
else:
SystemRandom().shuffle(blockPeers)
peerUsed = blockPeers.pop(0)
if not kv.get('shutdown') and peerUsed.strip() != '':
logger.info(
f"Attempting to download %s from {peerUsed}..." % (blockHash[:12],))
content = peeraction.peer_action(
shared_state, peerUsed,
'getdata/' + blockHash,
max_resp_size=3000000) # block content from random peer
if content is not False and len(content) > 0:
try:
content = content.encode()
except AttributeError:
pass
realHash = onionrcrypto.hashers.sha3_hash(content)
try:
realHash = realHash.decode() # bytes on some versions for some reason
except AttributeError:
pass
if realHash == blockHash:
#content = content.decode() # decode here because sha3Hash needs bytes above
metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
metadata = metas[0]
try:
metadata_validation_result = \
validatemetadata.validate_metadata(metadata, metas[2])
except onionrexceptions.PlaintextNotSupported:
logger.debug(f"Not saving {blockHash} due to plaintext not enabled")
removeFromQueue = True
except onionrexceptions.DataExists:
metadata_validation_result = False
if metadata_validation_result: # check if metadata is valid, and verify nonce
if onionrcrypto.cryptoutils.verify_POW(content): # check if POW is enough/correct
logger.info('Attempting to save block %s...' % blockHash[:12])
try:
onionrstorage.set_data(content)
except onionrexceptions.DataExists:
logger.warn('Data is already set for %s ' % (blockHash,))
except onionrexceptions.DiskAllocationReached:
logger.error('Reached disk allocation allowance, cannot save block %s.' % (blockHash,))
removeFromQueue = False
else:
blockmetadb.add_to_block_DB(blockHash, dataSaved=True) # add block to meta db
blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database
spawn(
local_command,
f'/daemon-event/upload_event',
post=True,
is_json=True,
post_data={'block': blockHash}
)
else:
logger.warn('POW failed for block %s.' % (blockHash,))
else:
if blacklist.inBlacklist(realHash):
logger.warn('Block %s is blacklisted.' % (realHash,))
else:
logger.warn('Metadata for block %s is invalid.' % (blockHash,))
blacklist.addToDB(blockHash)
else:
# if block didn't meet expected hash
tempHash = onionrcrypto.hashers.sha3_hash(content) # lazy hack, TODO use var
try:
tempHash = tempHash.decode()
except AttributeError:
pass
# Punish peer for sharing invalid block (not always malicious, but is bad regardless)
onionrpeers.PeerProfiles(peerUsed).addScore(-50)
if tempHash != 'ed55e34cb828232d6c14da0479709bfa10a0923dca2b380496e6b2ed4f7a0253':
# Dumb hack for 404 response from peer. Don't log it if 404 since its likely not malicious or a critical error.
logger.warn(
'Block hash validation failed for ' +
blockHash + ' got ' + tempHash)
else:
removeFromQueue = False # Don't remove from queue if 404
if removeFromQueue:
try:
del kv.get('blockQueue')[blockHash] # remove from block queue both if success or false
if count == LOG_SKIP_COUNT:
logger.info('%s blocks remaining in queue' %
[len(kv.get('blockQueue'))], terminal=True)
count = 0
except KeyError:
pass
kv.get('currentDownloading').remove(blockHash)

View File

@ -1,42 +0,0 @@
"""Onionr - Private P2P Communication.
Check if a block should be downloaded
(if we already have it or its blacklisted or not)
"""
from coredb import blockmetadb
from onionrblocks import onionrblacklist
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def should_download(shared_state, block_hash) -> bool:
"""Return bool for if a (assumed to exist) block should be downloaded."""
blacklist = onionrblacklist.OnionrBlackList()
should = True
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
if block_hash in blockmetadb.get_block_list():
# Don't download block we have
should = False
else:
if blacklist.inBlacklist(block_hash):
# Don't download blacklisted block
should = False
if should is False:
# Remove block from communicator queue if it shouldn't be downloaded
try:
del kv.get('blockQueue')[block_hash]
except KeyError:
pass
return should

View File

@ -1,108 +0,0 @@
"""Onionr - Private P2P Communication.
Cleanup old Onionr blocks and forward secrecy keys using the communicator.
Ran from a communicator timer usually
"""
import sqlite3
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
import logger
from onionrusers import onionrusers
from onionrutils import epoch
from coredb import blockmetadb, dbfiles
import onionrstorage
from onionrstorage import removeblock
from onionrblocks import onionrblacklist
from onionrblocks.storagecounter import StorageCounter
from etc.onionrvalues import DATABASE_LOCK_TIMEOUT
from onionrproofs import hashMeetsDifficulty
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
storage_counter = StorageCounter()
def __remove_from_upload(shared_state, block_hash: str):
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
try:
kv.get('blocksToUpload').remove(block_hash)
except ValueError:
pass
def __purge_block(shared_state, block_hash, add_to_blacklist = True):
blacklist = None
removeblock.remove_block(block_hash)
onionrstorage.deleteBlock(block_hash)
__remove_from_upload(shared_state, block_hash)
if add_to_blacklist:
blacklist = onionrblacklist.OnionrBlackList()
blacklist.addToDB(block_hash)
def clean_old_blocks(shared_state):
"""Delete expired blocks + old blocks if disk allocation is near full"""
blacklist = onionrblacklist.OnionrBlackList()
# Delete expired blocks
for bHash in blockmetadb.expiredblocks.get_expired_blocks():
__purge_block(shared_state, bHash, add_to_blacklist=True)
logger.info('Deleted expired block: %s' % (bHash,))
while storage_counter.is_full():
try:
oldest = blockmetadb.get_block_list()[0]
except IndexError:
break
else:
__purge_block(shared_state, bHash, add_to_blacklist=True)
logger.info('Deleted block because of full storage: %s' % (oldest,))
def clean_keys():
"""Delete expired forward secrecy keys"""
conn = sqlite3.connect(dbfiles.user_id_info_db,
timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
time = epoch.get_epoch()
deleteKeys = []
for entry in c.execute(
"SELECT * FROM forwardKeys WHERE expire <= ?", (time,)):
logger.debug('Forward key: %s' % entry[1])
deleteKeys.append(entry[1])
for key in deleteKeys:
logger.debug('Deleting forward key %s' % key)
c.execute("DELETE from forwardKeys where forwardKey = ?", (key,))
conn.commit()
conn.close()
onionrusers.deleteExpiredKeys()
def clean_blocks_not_meeting_pow(shared_state):
"""Clean blocks not meeting min send/rec pow. Used if config.json POW changes"""
block_list = blockmetadb.get_block_list()
for block in block_list:
if not hashMeetsDifficulty(block):
logger.warn(
f"Deleting block {block} because it was stored" +
"with a POW level smaller than current.", terminal=True)
__purge_block(shared_state, block)

View File

@ -1,66 +0,0 @@
"""Onionr - Private P2P Communication.
Lookup new peer transport addresses using the communicator
"""
from typing import TYPE_CHECKING
import logger
from onionrutils import stringvalidators
from communicator import peeraction, onlinepeers
from utils import gettransports
import onionrexceptions
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def lookup_new_peer_transports_with_communicator(shared_state):
logger.info('Looking up new addresses...')
tryAmount = 1
newPeers = []
transports = gettransports.get()
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
for i in range(tryAmount):
# Download new peer address list from random online peers
if len(newPeers) > 10000:
# Don't get new peers if we have too many queued up
break
try:
peer = onlinepeers.pick_online_peer(kv)
newAdders = peeraction.peer_action(shared_state, peer, action='pex')
except onionrexceptions.OnlinePeerNeeded:
continue
try:
newPeers = newAdders.split(',')
except AttributeError:
pass
else:
# Validate new peers are good format and not already in queue
invalid = []
for x in newPeers:
x = x.strip()
if not stringvalidators.validate_transport(x) \
or x in kv.get('newPeers') or x in transports:
# avoid adding if its our address
invalid.append(x)
for x in invalid:
try:
newPeers.remove(x)
except ValueError:
pass
kv.get('newPeers').extend(newPeers)

View File

@ -1,126 +0,0 @@
"""Onionr - Private P2P Communication.
Lookup new blocks with the communicator using a random connected peer
"""
from typing import TYPE_CHECKING
from gevent import time
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
import logger
import onionrproofs
from onionrutils import stringvalidators, epoch
from communicator import peeraction, onlinepeers
from coredb.blockmetadb import get_block_list
from utils import reconstructhash
from onionrblocks import onionrblacklist
import onionrexceptions
import config
from etc import onionrvalues
from onionrblocks.storagecounter import StorageCounter
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
blacklist = onionrblacklist.OnionrBlackList()
storage_counter = StorageCounter()
def lookup_blocks_from_communicator(shared_state: 'TooMany'):
logger.info('Looking up new blocks')
tryAmount = 2
newBlocks = ''
# List of existing saved blocks
existingBlocks = get_block_list()
triedPeers = [] # list of peers we've tried this time around
# Max amount of *new* block hashes to have in queue
maxBacklog = 1560
lastLookupTime = 0 # Last time we looked up a particular peer's list
new_block_count = 0
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
for i in range(tryAmount):
# Defined here to reset it each time, time offset is added later
listLookupCommand = 'getblocklist'
if len(kv.get('blockQueue')) >= maxBacklog:
break
if not kv.get('isOnline'):
break
# check if disk allocation is used
if storage_counter.is_full():
logger.debug(
'Not looking up new blocks due to maximum amount of disk used')
break
try:
# select random online peer
peer = onlinepeers.pick_online_peer(kv)
except onionrexceptions.OnlinePeerNeeded:
time.sleep(1)
continue
# if we've already tried all the online peers this time around, stop
if peer in triedPeers:
if len(kv.get('onlinePeers')) == len(triedPeers):
break
else:
continue
triedPeers.append(peer)
# Get the last time we looked up a peer's stamp,
# to only fetch blocks since then.
# Saved in memory only for privacy reasons
try:
lastLookupTime = kv.get('dbTimestamps')[peer]
except KeyError:
lastLookupTime = epoch.get_epoch() - onionrvalues.DEFAULT_EXPIRE
listLookupCommand += '?date=%s' % (lastLookupTime,)
try:
newBlocks = peeraction.peer_action(
shared_state,
peer, listLookupCommand) # get list of new block hashes
except Exception as error:
logger.warn(
f'Could not get new blocks from {peer}.',
error=error)
newBlocks = False
if newBlocks != False: # noqa
# if request was a success
for i in newBlocks.split('\n'):
if stringvalidators.validate_hash(i):
i = reconstructhash.reconstruct_hash(i)
# if newline seperated string is valid hash
# if block does not exist on disk + is not already in queue
if i not in existingBlocks:
if i not in kv.get('blockQueue'):
if onionrproofs.hashMeetsDifficulty(i) and \
not blacklist.inBlacklist(i):
if len(kv.get('blockQueue')) <= 1000000:
# add blocks to download queue
kv.get('blockQueue')[i] = [peer]
new_block_count += 1
kv.get('dbTimestamps')[peer] = \
epoch.get_rounded_epoch(roundS=60)
else:
if peer not in kv.get('blockQueue')[i]:
if len(kv.get('blockQueue')[i]) < 10:
kv.get('blockQueue')[i].append(peer)
if new_block_count > 0:
block_string = ""
if new_block_count > 1:
block_string = "s"
logger.info(
f'Discovered {new_block_count} new block{block_string}',
terminal=True)

View File

@ -1,64 +0,0 @@
"""
Onionr - Private P2P Communication.
Determine if our node is able to use Tor based
on the status of a communicator instance
and the result of pinging onion http servers
"""
import logger
from utils import netutils
from onionrutils import localcommand, epoch
from . import restarttor
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def net_check(shared_state):
"""Check if we are connected to the internet.
or not when we can't connect to any peers
"""
# for detecting if we have received incoming connections recently
rec = False
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
proxy_port = shared_state.get_by_string("NetController").socksPort
if len(kv.get('onlinePeers')) == 0:
try:
if (epoch.get_epoch() - int(localcommand.local_command
('/lastconnect'))) <= 60:
kv.put('isOnline', True)
rec = True
except ValueError:
pass
if not rec and not netutils.check_network(torPort=proxy_port):
if not kv.get('shutdown'):
if not shared_state.get_by_string(
"OnionrCommunicatorDaemon").config.get(
'general.offline_mode', False):
logger.warn('Network check failed, are you connected to ' +
'the Internet, and is Tor working? ' +
'This is usually temporary, but bugs and censorship can cause this to persist, in which case you should report it to beardog [at] mailbox.org', # noqa
terminal=True)
restarttor.restart(shared_state)
kv.put('offlinePeers', [])
kv.put('isOnline', False)
else:
kv.put('isOnline', True)

View File

@ -1,28 +0,0 @@
"""
Onionr - Private P2P Communication.
Pick a proxy to use based on a peer's address
"""
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def pick_proxy(peer_address):
if peer_address.endswith('.onion'):
return 'tor'
elif peer_address.endswith('.i2p'):
return 'i2p'
raise ValueError(
f"Peer address not ending with acceptable domain: {peer_address}")

View File

@ -1,28 +0,0 @@
"""
Onionr - Private P2P Communication.
Restart Onionr managed Tor
"""
import netcontroller
import config
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def restart(shared_state):
if not config.get('tor.use_existing_tor', False):
net = shared_state.get(netcontroller.NetController)
net.killTor()
net.startTor()

View File

@ -1,148 +0,0 @@
"""Onionr - Private P2P Communication.
Upload blocks in the upload queue to peers from the communicator
"""
from typing import TYPE_CHECKING
from time import sleep
from threading import Thread
from secrets import SystemRandom
from . import sessionmanager
from onionrtypes import UserID
import logger
from communicatorutils import proxypicker
import onionrexceptions
from onionrblocks import onionrblockapi as block
from onionrblocks.blockmetadata.fromdata import get_block_metadata_from_data
from onionrutils import stringvalidators, basicrequests
from onionrutils.validatemetadata import validate_metadata
from communicator import onlinepeers
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
from communicator import OnionrCommunicatorDaemon
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def upload_blocks_from_communicator(shared_state: 'OnionrCommunicatorDaemon'):
"""Accept a communicator instance + upload blocks from its upload queue."""
"""when inserting a block, we try to upload
it to a few peers to add some deniability & increase functionality"""
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
session_manager: sessionmanager.BlockUploadSessionManager
session_manager = shared_state.get(
sessionmanager.BlockUploadSessionManager)
tried_peers: UserID = []
finishedUploads = []
SystemRandom().shuffle(kv.get('blocksToUpload'))
def remove_from_hidden(bl):
sleep(60)
try:
shared_state.get_by_string(
'PublicAPI').hideBlocks.remove(bl)
except ValueError:
pass
if len(kv.get('blocksToUpload')) != 0:
for bl in kv.get('blocksToUpload'):
if not stringvalidators.validate_hash(bl):
logger.warn('Requested to upload invalid block', terminal=True)
return
session = session_manager.add_session(bl)
for _ in range(min(len(kv.get('onlinePeers')), 6)):
try:
peer = onlinepeers.pick_online_peer(kv)
if not block.Block(bl).isEncrypted:
if peer in kv.get('plaintextDisabledPeers'):
logger.info(f"Cannot upload plaintext block to peer that denies it {peer}") # noqa
continue
except onionrexceptions.OnlinePeerNeeded:
continue
try:
session.peer_exists[peer]
continue
except KeyError:
pass
try:
if session.peer_fails[peer] > 3:
continue
except KeyError:
pass
if peer in tried_peers:
continue
tried_peers.append(peer)
url = f'http://{peer}/upload'
try:
data = block.Block(bl).getRaw()
if not data:
logger.warn(
f"Couldn't get data for block in upload list {bl}",
terminal=True)
raise onionrexceptions.NoDataAvailable
try:
def __check_metadata():
metadata = get_block_metadata_from_data(data)[0]
if not validate_metadata(metadata, data):
logger.warn(
f"Metadata for uploading block not valid {bl}")
raise onionrexceptions.InvalidMetadata
__check_metadata()
except onionrexceptions.DataExists:
pass
except( # noqa
onionrexceptions.NoDataAvailable,
onionrexceptions.InvalidMetadata) as _:
finishedUploads.append(bl)
break
proxy_type = proxypicker.pick_proxy(peer)
logger.info(
f"Uploading block {bl[:8]} to {peer}", terminal=True)
resp = basicrequests.do_post_request(
url, data=data, proxyType=proxy_type,
content_type='application/octet-stream')
if resp is not False:
if resp == 'success':
Thread(target=remove_from_hidden,
args=[bl], daemon=True).start()
session.success()
session.peer_exists[peer] = True
elif resp == 'exists':
session.success()
session.peer_exists[peer] = True
else:
session.fail()
session.fail_peer(peer)
shared_state.get_by_string(
'OnionrCommunicatorDaemon').getPeerProfileInstance(
peer).addScore(-5)
logger.warn(
f'Failed to upload {bl[:8]}, reason: {resp}',
terminal=True)
else:
session.fail()
session_manager.clean_session()
for x in finishedUploads:
try:
kv.get('blocksToUpload').remove(x)
shared_state.get_by_string(
'PublicAPI').hideBlocks.remove(x)
except ValueError:
pass

View File

@ -1,48 +0,0 @@
"""Onionr - Private P2P Communication.
Perform block mixing
"""
import time
from typing import List
import onionrtypes
from onionrblocks import onionrblockapi
from .pool import UploadPool
from .pool import PoolFullException
from etc import onionrvalues
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
upload_pool = UploadPool(4)
def block_mixer(upload_list: List[onionrtypes.BlockHash],
block_to_mix: onionrtypes.BlockHash):
"""Delay and mix block inserts.
Take a block list and a received/created block and add it
to the said block list
"""
bl = onionrblockapi.Block(block_to_mix)
try:
if time.time() - bl.claimedTime > onionrvalues.BLOCK_POOL_MAX_AGE:
raise ValueError
except TypeError:
pass
if block_to_mix:
upload_list.append(block_to_mix)

View File

@ -1,71 +0,0 @@
"""Onionr - Private P2P Communication.
Upload pool
"""
from typing import List
from secrets import SystemRandom
import onionrutils
import onionrtypes
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
class PoolFullException(Exception):
"""For when the UploadPool is full.
Raise when a new hash is attempted to be added
"""
class PoolNotReady(Exception):
"""Raise when UploadPool pool access is attempted without it being full."""
class AlreadyInPool(Exception):
"""Raise when a hash already in pool is attempted to be added again."""
class UploadPool:
"""Upload pool for mixing blocks together and delaying uploads."""
def __init__(self, pool_size: int):
"""Create a new pool with a specified max size.
Uses private var and getter to avoid direct adding
"""
self._pool: List[onionrtypes.BlockHash] = []
self._pool_size = pool_size
self.birthday = onionrutils.epoch.get_epoch()
def add_to_pool(self, item: List[onionrtypes.BlockHash]):
"""Add a new hash to the pool. Raise PoolFullException if full."""
if len(self._pool) >= self._pool_size:
raise PoolFullException
if not onionrutils.stringvalidators.validate_hash(item):
raise ValueError
self._pool.append(item)
def get_pool(self) -> List[onionrtypes.BlockHash]:
"""Get the hash pool in secure random order."""
if len(self._pool) != self._pool_size:
raise PoolNotReady
final_pool: List[onionrtypes.BlockHash] = list(self._pool)
SystemRandom().shuffle(final_pool)
self._pool.clear()
self.birthday = onionrutils.epoch.get_epoch()
return final_pool

View File

@ -1,57 +0,0 @@
"""Onionr - Private P2P Communication.
Virtual upload "sessions" for blocks
"""
from typing import Union, Dict
from onionrtypes import UserID
from onionrutils import stringvalidators
from onionrutils import bytesconverter
from onionrutils import epoch
from utils import reconstructhash
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
class UploadSession:
"""Manage statistics for an Onionr block upload session.
accept a block hash (incl. unpadded) as an argument
"""
def __init__(self, block_hash: Union[str, bytes]):
block_hash = bytesconverter.bytes_to_str(block_hash)
block_hash = reconstructhash.reconstruct_hash(block_hash)
if not stringvalidators.validate_hash(block_hash):
raise ValueError
self.start_time = epoch.get_epoch()
self.block_hash = reconstructhash.deconstruct_hash(block_hash)
self.total_fail_count: int = 0
self.total_success_count: int = 0
self.peer_fails: Dict[UserID, int] = {}
self.peer_exists: Dict[UserID, bool] = {}
def fail_peer(self, peer):
try:
self.peer_fails[peer] += 1
except KeyError:
self.peer_fails[peer] = 0
def fail(self):
self.total_fail_count += 1
def success(self):
self.total_success_count += 1

View File

@ -1,127 +0,0 @@
"""Onionr - Private P2P Communication.
Manager for upload 'sessions'
"""
from typing import List, Union, TYPE_CHECKING
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
from session import UploadSession
from onionrutils import bytesconverter
from etc import onionrvalues
from utils import reconstructhash
from . import session
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
class BlockUploadSessionManager:
"""Holds block UploadSession instances.
Optionally accepts iterable of sessions to added on init
Arguments: old_session: iterable of old UploadSession objects
"""
def __init__(self, old_sessions: List = None):
if old_sessions is None:
self.sessions = []
else:
self.sessions = old_sessions
def add_session(self,
session_or_block: Union[str,
bytes,
session.UploadSession
]
) -> session.UploadSession:
"""Create (or add existing) block upload session.
from a str/bytes block hex hash, existing UploadSession
"""
if isinstance(session_or_block, session.UploadSession):
if session_or_block not in self.sessions:
self.sessions.append(session_or_block)
return session_or_block
try:
return self.get_session(session_or_block)
except KeyError:
pass
# convert bytes hash to str
if isinstance(session_or_block, bytes):
session_or_block = bytesconverter.bytes_to_str(session_or_block)
# intentionally not elif
if isinstance(session_or_block, str):
new_session = session.UploadSession(session_or_block)
self.sessions.append(new_session)
return new_session
raise ValueError
def get_session(self,
block_hash: Union[str, bytes]
) -> session.UploadSession:
block_hash = reconstructhash.deconstruct_hash(
bytesconverter.bytes_to_str(block_hash))
for sess in self.sessions:
if sess.block_hash == block_hash:
return sess
raise KeyError
def clean_session(self,
specific_session: Union[str, 'UploadSession'] = None):
comm_inst: 'OnionrCommunicatorDaemon' # type: ignore
comm_inst = self._too_many.get_by_string( # pylint: disable=E1101 type: ignore
"OnionrCommunicatorDaemon")
kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string(
"DeadSimpleKV")
sessions_to_delete = []
if kv.get('startTime') < 120:
return
onlinePeerCount = len(kv.get('onlinePeers'))
# If we have no online peers right now,
if onlinePeerCount == 0:
return
for sess in self.sessions:
# if over 50% of peers that were online for a session have
# become unavailable, don't kill sessions
if sess.total_success_count > onlinePeerCount:
if onlinePeerCount / sess.total_success_count >= 0.5:
return
# Clean sessions if they have uploaded to enough online peers
if sess.total_success_count <= 0:
continue
if (sess.total_success_count / onlinePeerCount) >= \
onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT:
sessions_to_delete.append(sess)
for sess in sessions_to_delete:
try:
self.sessions.remove(session)
except ValueError:
pass
# TODO cleanup to one round of search
# Remove the blocks from the sessions, upload list,
# and waitforshare list
try:
kv.get('blocksToUpload').remove(
reconstructhash.reconstruct_hash(sess.block_hash))
except ValueError:
pass
try:
kv.get('blocksToUpload').remove(sess.block_hash)
except ValueError:
pass

View File

@ -1 +0,0 @@
from . import keydb, blockmetadb

View File

@ -1,84 +0,0 @@
"""Onionr - Private P2P Communication.
Work with information relating to blocks stored on the node
"""
import sqlite3
from etc import onionrvalues
from . import expiredblocks, updateblockinfo, add
from .. import dbfiles
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
update_block_info = updateblockinfo.update_block_info
add_to_block_DB = add.add_to_block_DB
def get_block_list(date_rec=None, unsaved=False):
"""Get list of our blocks."""
if date_rec is None:
date_rec = 0
conn = sqlite3.connect(
dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
execute = 'SELECT hash FROM hashes WHERE dateReceived' + \
' >= ? ORDER BY dateReceived ASC;'
args = (date_rec,)
rows = list()
for row in c.execute(execute, args):
for i in row:
rows.append(i)
conn.close()
return rows
def get_block_date(blockHash):
"""Return the date a block was received."""
conn = sqlite3.connect(
dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
execute = 'SELECT dateReceived FROM hashes WHERE hash=?;'
args = (blockHash,)
for row in c.execute(execute, args):
for i in row:
return int(i)
conn.close()
return None
def get_blocks_by_type(blockType, orderDate=True):
"""Return a list of blocks by the type."""
conn = sqlite3.connect(
dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
if orderDate:
execute = 'SELECT hash FROM hashes WHERE dataType=? ORDER BY dateReceived;'
else:
execute = 'SELECT hash FROM hashes WHERE dataType=?;'
args = (blockType,)
rows = list()
for row in c.execute(execute, args):
for i in row:
rows.append(i)
conn.close()
return rows

View File

@ -1,49 +0,0 @@
"""Onionr - Private P2P Communication.
Add an entry to the block metadata database
"""
import sqlite3
import secrets
from onionrutils import epoch
from onionrblocks import blockmetadata
from etc import onionrvalues
from .. import dbfiles
from onionrexceptions import BlockMetaEntryExists
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def add_to_block_DB(newHash, selfInsert=False, dataSaved=False):
"""
Add a hash value to the block db
Should be in hex format!
"""
if blockmetadata.has_block(newHash):
raise BlockMetaEntryExists
conn = sqlite3.connect(
dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
currentTime = epoch.get_epoch() + secrets.randbelow(61)
if selfInsert or dataSaved:
selfInsert = 1
else:
selfInsert = 0
data = (newHash, currentTime, '', selfInsert)
c.execute(
'INSERT INTO hashes (hash, dateReceived, dataType, dataSaved) VALUES(?, ?, ?, ?);', data)
conn.commit()
conn.close()

View File

@ -1,41 +0,0 @@
"""Onionr - Private P2P Communication.
Get a list of expired blocks still stored
"""
import sqlite3
from onionrutils import epoch
from .. import dbfiles
from etc import onionrvalues
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def get_expired_blocks():
"""Return a list of expired blocks."""
conn = sqlite3.connect(
dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
date = int(epoch.get_epoch())
compiled = (date,)
execute = 'SELECT hash FROM hashes WHERE ' + \
'expire <= ? ORDER BY dateReceived;'
rows = list()
for row in c.execute(execute, compiled):
for i in row:
rows.append(i)
conn.close()
return rows

View File

@ -1,52 +0,0 @@
"""Onionr - Private P2P Communication.
Update block information in the metadata database by a field name
"""
import sqlite3
from .. import dbfiles
from etc import onionrvalues
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def update_block_info(hash, key, data):
"""set info associated with a block
hash - the hash of a block
dateReceived - the date the block was recieved, not necessarily when it was created
decrypted - if we can successfully decrypt the block
dataType - data type of the block
dataFound - if the data has been found for the block
dataSaved - if the data has been saved for the block
sig - defunct
author - defunct
dateClaimed - timestamp claimed inside the block, only as trustworthy as the block author is
expire - expire date for a block
"""
if key not in ('dateReceived', 'decrypted', 'dataType', 'dataFound',
'dataSaved', 'sig', 'author', 'dateClaimed', 'expire'):
raise ValueError('Key must be in the allowed list')
conn = sqlite3.connect(dbfiles.block_meta_db,
timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
args = (data, hash)
# Unfortunately, not really possible to prepare this statement
c.execute("UPDATE hashes SET " + key + " = ? where hash = ?;", args)
conn.commit()
conn.close()
return True

View File

@ -1,11 +0,0 @@
from utils import identifyhome
import filepaths
home = identifyhome.identify_home()
if not home.endswith('/'): home += '/'
block_meta_db = '%sblock-metadata.db' % (home)
block_data_db = '%s/block-data.db' % (filepaths.block_data_location,)
address_info_db = '%saddress.db' % (home,)
user_id_info_db = '%susers.db' % (home,)
forward_keys_db = '%sforward-keys.db' % (home,)
blacklist_db = '%sblacklist.db' % (home,)

View File

@ -1 +0,0 @@
from . import addkeys, listkeys, removekeys, userinfo, transportinfo

View File

@ -1,88 +0,0 @@
"""Onionr - Private P2P Communication.
add user keys or transport addresses
"""
import sqlite3
from onionrutils import stringvalidators
from . import listkeys
from utils import gettransports
from .. import dbfiles
import onionrcrypto
from etc import onionrvalues
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def add_peer(peerID, name=''):
"""Add a public key to the key database (misleading function name)."""
if peerID in listkeys.list_peers() or peerID == onionrcrypto.pub_key:
raise ValueError("specified id is already known")
# This function simply adds a peer to the DB
if not stringvalidators.validate_pub_key(peerID):
return False
conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
hashID = ""
c = conn.cursor()
t = (peerID, name, 'unknown', hashID, 0)
for i in c.execute("SELECT * FROM peers WHERE id = ?;", (peerID,)):
try:
if i[0] == peerID:
conn.close()
return False
except ValueError:
pass
except IndexError:
pass
c.execute('INSERT INTO peers (id, name, dateSeen, hashID, trust) VALUES(?, ?, ?, ?, ?);', t)
conn.commit()
conn.close()
return True
def add_address(address):
"""Add an address to the address database (only tor currently)"""
if type(address) is None or len(address) == 0:
return False
if stringvalidators.validate_transport(address):
if address in gettransports.get():
return False
conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
# check if address is in database
# this is safe to do because the address is validated above, but we strip some chars here too just in case
address = address.replace('\'', '').replace(';', '').replace('"', '').replace('\\', '')
for i in c.execute("SELECT * FROM adders WHERE address = ?;", (address,)):
try:
if i[0] == address:
conn.close()
return False
except ValueError:
pass
except IndexError:
pass
t = (address, 1)
c.execute('INSERT INTO adders (address, type) VALUES(?, ?);', t)
conn.commit()
conn.close()
return True
else:
return False

View File

@ -1,86 +0,0 @@
'''
Onionr - Private P2P Communication
get lists for user keys or transport addresses
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import sqlite3
import logger
from onionrutils import epoch
from etc import onionrvalues
from .. import dbfiles
from . import userinfo, transportinfo
def list_peers(randomOrder=True, getPow=False, trust=0):
'''
Return a list of public keys (misleading function name)
randomOrder determines if the list should be in a random order
trust sets the minimum trust to list
'''
conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
payload = ''
if trust not in (0, 1, 2):
logger.error('Tried to select invalid trust.')
return
if randomOrder:
payload = 'SELECT * FROM peers WHERE trust >= ? ORDER BY RANDOM();'
else:
payload = 'SELECT * FROM peers WHERE trust >= ?;'
peerList = []
for i in c.execute(payload, (trust,)):
try:
if len(i[0]) != 0:
if getPow:
peerList.append(i[0] + '-' + i[1])
else:
peerList.append(i[0])
except TypeError:
pass
conn.close()
return peerList
def list_adders(randomOrder=True, i2p=True, recent=0):
'''
Return a list of transport addresses
'''
conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
if randomOrder:
addresses = c.execute('SELECT * FROM adders ORDER BY RANDOM();')
else:
addresses = c.execute('SELECT * FROM adders;')
addressList = []
for i in addresses:
if len(i[0].strip()) == 0:
continue
addressList.append(i[0])
conn.close()
testList = list(addressList) # create new list to iterate
for address in testList:
try:
if recent > 0 and (epoch.get_epoch() - transportinfo.get_address_info(address, 'lastConnect')) > recent:
raise TypeError # If there is no last-connected date or it was too long ago, don't add peer to list if recent is not 0
except TypeError:
addressList.remove(address)
return addressList

View File

@ -1,60 +0,0 @@
'''
Onionr - Private P2P Communication
Remove a transport address but don't ban them
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import sqlite3
from onionrplugins import onionrevents as events
from onionrutils import stringvalidators
from onionrutils import mnemonickeys
from .. import dbfiles
from etc import onionrvalues
def remove_address(address):
'''
Remove an address from the address database
'''
if stringvalidators.validate_transport(address):
conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
t = (address,)
c.execute('Delete from adders where address=?;', t)
conn.commit()
conn.close()
#events.event('address_remove', data = {'address': address}, onionr = core_inst.onionrInst)
return True
else:
return False
def remove_user(pubkey: str)->bool:
'''
Remove a user from the user database
'''
pubkey = mnemonickeys.get_base32(pubkey)
if stringvalidators.validate_pub_key(pubkey):
conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
t = (pubkey,)
c.execute('Delete from peers where id=?;', t)
conn.commit()
conn.close()
return True
else:
return False

View File

@ -1,85 +0,0 @@
"""Onionr - Private P2P Communication.
get or set transport address meta information
"""
import sqlite3
from .. import dbfiles
from etc import onionrvalues
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
info_numbers = {
'address': 0,
'type': 1,
'knownPeer': 2,
'speed': 3,
'success': 4,
'powValue': 5,
'failure': 6,
'lastConnect': 7,
'trust': 8,
'introduced': 9}
def get_address_info(address, info):
"""Get info about an address from its database entry.
address text, 0
type int, 1
knownPeer text, 2
speed int, 3
success int, 4
powValue 5
failure int 6
lastConnect 7
trust 8
introduced 9
"""
conn = sqlite3.connect(
dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
command = (address,)
info = info_numbers[info]
iter_count = 0
retVal = ''
for row in c.execute('SELECT * FROM adders WHERE address=?;', command):
for i in row:
if iter_count == info:
retVal = i
break
else:
iter_count += 1
conn.close()
return retVal
def set_address_info(address, key, data):
"""Update an address for a key."""
conn = sqlite3.connect(
dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
command = (data, address)
if key not in info_numbers.keys():
raise ValueError(
"Got invalid database key when setting address info, must be in whitelist")
else:
c.execute('UPDATE adders SET ' + key + ' = ? WHERE address=?', command)
conn.commit()
conn.close()

View File

@ -1,73 +0,0 @@
'''
Onionr - Private P2P Communication
get or set information about a user id
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import sqlite3
from .. import dbfiles
from etc import onionrvalues
def get_user_info(peer, info):
'''
Get info about a peer from their database entry
id text 0
name text, 1
adders text, 2
dateSeen not null, 3
trust int 4
hashID text 5
'''
conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
command = (peer,)
infoNumbers = {'id': 0, 'name': 1, 'adders': 2, 'dateSeen': 3, 'trust': 4, 'hashID': 5}
info = infoNumbers[info]
iterCount = 0
retVal = ''
for row in c.execute('SELECT * FROM peers WHERE id=?;', command):
for i in row:
if iterCount == info:
retVal = i
break
else:
iterCount += 1
conn.close()
return retVal
def set_peer_info(peer, key, data):
'''
Update a peer for a key
'''
conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
command = (data, peer)
if key not in ('id', 'name', 'pubkey', 'forwardKey', 'dateSeen', 'trust'):
raise ValueError("Got invalid database key when setting peer info")
c.execute('UPDATE peers SET ' + key + ' = ? WHERE id=?', command)
conn.commit()
conn.close()
set_user_info = set_peer_info

View File

@ -33,7 +33,6 @@ def delete_run_files():
Test: test_cleanup.py
"""
_safe_remove(filepaths.public_API_host_file)
_safe_remove(filepaths.private_API_host_file)
_safe_remove(filepaths.daemon_mark_file)
_safe_remove(filepaths.lock_file)

View File

@ -1 +1,13 @@
from urllib3.contrib.socks import SOCKSProxyManager # noqa
from urllib3.contrib.socks import SOCKSProxyManager # noqa
import gevent
import flask
import mimcvdf
import toomanyobjs
import stem
import psutil
import unpaddedbase32
import niceware
import watchdog
import nacl
import deadsimplekv
import ujson

View File

@ -1,22 +1,24 @@
'''
Onionr - Private P2P Communication
"""
Onionr - Private P2P Communication.
human_readable_time takes integer seconds and returns a human readable string
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
human_readable_time takes integer seconds and returns a human readable string
"""
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
def human_readable_time(seconds):
build = ''

View File

@ -20,23 +20,16 @@ import filepaths
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
DENIABLE_PEER_ADDRESS = "OVPCZLOXD6DC5JHX4EQ3PSOGAZ3T24F75HQLIUZSDSMYPEOXCPFA"
PASSWORD_LENGTH = 25
ONIONR_TAGLINE = 'Private P2P Communication - GPLv3 - https://Onionr.net'
ONIONR_VERSION = '8.0.2'
ONIONR_VERSION_CODENAME = 'Genesis'
ONIONR_VERSION = '9.0.0'
ONIONR_VERSION_CODENAME = 'Nexus'
ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION)
API_VERSION = '2' # increments of 1; only change when something fundamental about how the API works changes. This way other nodes know how to communicate without learning too much information about you.
MIN_PY_VERSION = 7 # min version of 7 so we can take advantage of non-cyclic type hints
DEVELOPMENT_MODE = False
IS_QUBES = False
"""limit type length for a block (soft enforced, ignored if invalid but block still stored)."""
MAX_BLOCK_TYPE_LENGTH = 15
"""limit clock timestamp for new blocks to be skewed in the future in seconds,
2 minutes to allow plenty of time for slow block insertion and slight clock inaccuracies"""
MAX_BLOCK_CLOCK_SKEW = 120
"""Onionr user IDs are ed25519 keys, which are always 32 bytes in length"""
MAIN_PUBLIC_KEY_SIZE = 32
ORIG_RUN_DIR_ENV_VAR = 'ORIG_ONIONR_RUN_DIR'
DATABASE_LOCK_TIMEOUT = 60
@ -46,27 +39,9 @@ MIN_BLOCK_UPLOAD_PEER_PERCENT = 0.1
WSGI_SERVER_REQUEST_TIMEOUT_SECS = 120
MAX_NEW_PEER_QUEUE = 1000
BLOCK_EXPORT_FILE_EXT = '.onionr'
# Begin OnionrValues migrated values
"""30 days is plenty of time for someone to decide to renew a block"""
DEFAULT_EXPIRE = 2678400
# Metadata header section length limits, in bytes
BLOCK_METADATA_LENGTHS = {'meta': 1000, 'sig': 200, 'signer': 200, 'time': 10,
'n': 1000, 'c': 1000, 'encryptType': 4, 'expire': 14}
# Pool Eligibility Max Age
BLOCK_POOL_MAX_AGE = 300
"""Public key that signs MOTD messages shown in the web UI"""
MOTD_SIGN_KEY = "TRH763JURNY47QPBTTQ4LLPYCYQK6Q5YA33R6GANKZK5C5DKCIGQ"
"""Public key that signs update notifications."""
UPDATE_SIGN_KEY = "TRH763JURNY47QPBTTQ4LLPYCYQK6Q5YA33R6GANKZK5C5DKCIGQ"
if os.path.exists(filepaths.daemon_mark_file):
SCRIPT_NAME = 'start-daemon.sh'

View File

@ -6,33 +6,26 @@ if not home.endswith('/'): home += '/'
app_root = os.path.dirname(os.path.realpath(__file__)) + '/../../'
usage_file = home + 'disk-usage.txt'
block_data_location = home + 'blocks/'
contacts_location = home + 'contacts/'
public_API_host_file = home + 'public-host.txt'
private_API_host_file = home + 'private-host.txt'
bootstrap_file_location = 'static-data/bootstrap-nodes.txt'
data_nonce_file = home + 'block-nonces.dat'
forward_keys_file = home + 'forward-keys.db'
cached_storage = home + 'cachedstorage.dat'
announce_cache = home + 'announcecache.dat'
export_location = home + 'block-export/'
upload_list = home + 'upload-list.json'
config_file = home + 'config.json'
daemon_mark_file = home + '/daemon-true.txt'
lock_file = home + 'onionr.lock'
site_cache = home + 'onionr-sites.txt'
tor_hs_loc = home + 'hs/'
tor_hs_address_file = home + 'hs/hostname'
main_safedb = home + "main.safe.db"
data_nonce_file = home + 'block-nonces.dat'
keys_file = home + 'keys.txt'
onboarding_mark_file = home + 'onboarding-completed'
log_file = home + 'onionr.log'
ephemeral_services_file = home + 'ephemeral-services.list'
restarting_indicator = home + "is-restarting"
secure_erase_key_file = home + "erase-key"
master_db_location = home + "database/"

View File

@ -8,6 +8,4 @@ configapi: manage onionr configuration from the client http api
friendsapi: add, remove and list friends from the client http api
miscpublicapi: misculanious onionr network interaction from the **public** httpapi, such as announcements, block fetching and uploading.
profilesapi: work in progress in returning a profile page for an Onionr user

View File

@ -1,25 +1,29 @@
"""
Onionr - Private P2P Communication
Onionr - Private P2P Communication
This file registers plugin's flask blueprints for the client http server
"""
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
Register plugin's flask blueprints for the client http server
"""
import onionrplugins
import config
from . import wrappedfunctions
from . import fdsafehandler
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def load_plugin_blueprints(flaskapp, blueprint: str = 'flask_blueprint'):
"""Iterate enabled plugins and load any http endpoints they have"""
config.reload()

View File

@ -1,3 +1 @@
from . import shutdown, setbindip, getblockdata
GetBlockData = getblockdata.GetBlockData
from . import shutdown, setbindip

View File

@ -1,38 +0,0 @@
import ujson as json
from onionrblocks import onionrblockapi
from onionrutils import bytesconverter, stringvalidators
import onionrexceptions
class GetBlockData:
def __init__(self, client_api_inst=None):
return
def get_block_data(self, bHash, decrypt=False, raw=False, headerOnly=False):
if not stringvalidators.validate_hash(bHash):
raise onionrexceptions.InvalidHexHash(
"block hash not valid hash format")
bl = onionrblockapi.Block(bHash)
if decrypt:
bl.decrypt()
if bl.isEncrypted and not bl.decrypted:
raise ValueError
if not raw:
if not headerOnly:
retData = {'meta':bl.bheader, 'metadata': bl.bmetadata, 'content': bl.bcontent}
for x in list(retData.keys()):
try:
retData[x] = retData[x].decode()
except AttributeError:
pass
else:
validSig = False
signer = bytesconverter.bytes_to_str(bl.signer)
if bl.isSigned() and stringvalidators.validate_pub_key(signer) and bl.isSigner(signer):
validSig = True
bl.bheader['validSig'] = validSig
bl.bheader['meta'] = ''
retData = {'meta': bl.bheader, 'metadata': bl.bmetadata}
return json.dumps(retData)
else:
return bl.raw

View File

@ -24,7 +24,6 @@ shutdown_bp = Blueprint('shutdown', __name__)
def shutdown(client_api_inst):
try:
client_api_inst.publicAPI.httpServer.stop()
client_api_inst.httpServer.stop()
except AttributeError:
pass

View File

@ -1,75 +0,0 @@
"""Onionr - Private P2P Communication.
This file creates http endpoints for friend management
"""
import ujson as json
from onionrusers import contactmanager
from flask import Blueprint, Response, request, abort, redirect
from coredb import keydb
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
friends = Blueprint('friends', __name__)
@friends.route('/friends/list')
def list_friends():
pubkey_list = {}
friend_list = contactmanager.ContactManager.list_friends()
for friend in friend_list:
pubkey_list[friend.publicKey] = {'name': friend.get_info('name')}
return json.dumps(pubkey_list)
@friends.route('/friends/add/<pubkey>', methods=['POST'])
def add_friend(pubkey):
contactmanager.ContactManager(pubkey, saveUser=True).setTrust(1)
try:
return redirect(request.referrer + '#' + request.form['token'])
except TypeError:
return Response(
"Added, but referrer not set, cannot return to friends page")
@friends.route('/friends/remove/<pubkey>', methods=['POST'])
def remove_friend(pubkey):
contactmanager.ContactManager(pubkey).setTrust(0)
contactmanager.ContactManager(pubkey).delete_contact()
keydb.removekeys.remove_user(pubkey)
try:
return redirect(request.referrer + '#' + request.form['token'])
except TypeError:
return Response(
"Friend removed, but referrer not set, cannot return to page")
@friends.route('/friends/setinfo/<pubkey>/<key>', methods=['POST'])
def set_info(pubkey, key):
data = request.form['data']
contactmanager.ContactManager(pubkey).set_info(key, data)
try:
return redirect(request.referrer + '#' + request.form['token'])
except TypeError:
return Response(
"Info set, but referrer not set, cannot return to friends page")
@friends.route('/friends/getinfo/<pubkey>/<key>')
def get_info(pubkey, key):
ret_data = contactmanager.ContactManager(pubkey).get_info(key)
if ret_data is None:
abort(404)
else:
return ret_data

View File

@ -1,91 +0,0 @@
"""Onionr - Private P2P Communication.
Create blocks with the client api server
"""
import ujson as json
import threading
from typing import TYPE_CHECKING
from flask import Blueprint, Response, request, g
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
import onionrblocks
from onionrcrypto import hashers
from onionrutils import bytesconverter
from onionrutils import mnemonickeys
from onionrtypes import JSONSerializable
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
ib = Blueprint('insertblock', __name__)
@ib.route('/insertblock', methods=['POST'])
def client_api_insert_block():
insert_data: JSONSerializable = request.get_json(force=True)
message = insert_data['message']
message_hash = bytesconverter.bytes_to_str(hashers.sha3_hash(message))
kv: 'DeadSimpleKV' = g.too_many.get_by_string('DeadSimpleKV')
# Detect if message (block body) is not specified
if type(message) is None:
return 'failure due to unspecified message', 400
# Detect if block with same message is already being inserted
if message_hash in kv.get('generating_blocks'):
return 'failure due to duplicate insert', 400
else:
kv.get('generating_blocks').append(message_hash)
encrypt_type = ''
sign = True
meta = {}
to = ''
try:
if insert_data['encrypt']:
to = insert_data['to'].strip()
if "-" in to:
to = mnemonickeys.get_base32(to)
encrypt_type = 'asym'
except KeyError:
pass
try:
if not insert_data['sign']:
sign = False
except KeyError:
pass
try:
bType = insert_data['type']
except KeyError:
bType = 'bin'
try:
meta = json.loads(insert_data['meta'])
except KeyError:
pass
try:
# Setting in the mail UI is for if forward secrecy is *enabled*
disable_forward_secrecy = not insert_data['forward']
except KeyError:
disable_forward_secrecy = False
threading.Thread(
target=onionrblocks.insert, args=(message,),
kwargs={'header': bType, 'encryptType': encrypt_type,
'sign': sign, 'asymPeer': to, 'meta': meta,
'disableForward': disable_forward_secrecy}).start()
return Response('success')

View File

@ -1 +1 @@
from . import getblocks, staticfiles, endpoints, motd
from . import staticfiles, endpoints

View File

@ -1,31 +0,0 @@
"""Onionr - Private P2P Communication.
add a transport address to the db
"""
from onionrutils.stringvalidators import validate_transport
from coredb.keydb.addkeys import add_address
from coredb.keydb.listkeys import list_adders
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def add_peer(peer):
if peer in list_adders():
return "already added"
if add_address(peer):
return "success"
else:
return "failure, invalid address"

View File

@ -10,35 +10,28 @@ from sys import stdout as sys_stdout
from flask import Response, Blueprint, request, send_from_directory, abort
from flask import g
from gevent import sleep
import unpaddedbase32
from httpapi import apiutils
import onionrcrypto
import config
from netcontroller import NetController
from onionrstatistics.serializeddata import SerializedData
from onionrutils import mnemonickeys
from onionrutils import bytesconverter
from etc import onionrvalues
from utils import reconstructhash
from utils.gettransports import get as get_tor
from .addpeer import add_peer
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
pub_key = onionrcrypto.pub_key.replace('=', '')
SCRIPT_NAME = os.path.dirname(os.path.realpath(__file__)) + \
f'/../../../{onionrvalues.SCRIPT_NAME}'
@ -49,17 +42,6 @@ class PrivateEndpoints:
private_endpoints_bp = Blueprint('privateendpoints', __name__)
self.private_endpoints_bp = private_endpoints_bp
@private_endpoints_bp.route('/addpeer/<name>', methods=['post'])
def add_peer_endpoint(name):
result = add_peer(name)
if result == "success":
return Response("success")
else:
if "already" in result:
return Response(result, 409)
else:
return Response(result, 400)
@private_endpoints_bp.route('/www/<path:path>', endpoint='www')
def wwwPublic(path):
if not config.get("www.private.run", True):
@ -75,34 +57,11 @@ class PrivateEndpoints:
def get_is_atty():
return Response(str(sys_stdout.isatty()).lower())
@private_endpoints_bp.route('/hitcount')
def get_hit_count():
return Response(str(client_api.publicAPI.hitCount))
@private_endpoints_bp.route('/ping')
def ping():
# Used to check if client api is working
return Response("pong!")
@private_endpoints_bp.route('/lastconnect')
def last_connect():
return Response(str(client_api.publicAPI.lastRequest))
@private_endpoints_bp.route('/waitforshare/<name>', methods=['post'])
def wait_for_share(name):
"""Prevent the **public** api from sharing blocks.
Used for blocks we created usually
"""
if not name.isalnum():
raise ValueError('block hash needs to be alpha numeric')
name = reconstructhash.reconstruct_hash(name)
if name in client_api.publicAPI.hideBlocks:
return Response("will be removed")
else:
client_api.publicAPI.hideBlocks.append(name)
return Response("added")
@private_endpoints_bp.route('/shutdown')
def shutdown():
return apiutils.shutdown.shutdown(client_api)
@ -112,10 +71,6 @@ class PrivateEndpoints:
subprocess.Popen([SCRIPT_NAME, 'restart'])
return Response("bye")
@private_endpoints_bp.route('/gethidden')
def get_hidden_blocks():
return Response('\n'.join(client_api.publicAPI.hideBlocks))
@private_endpoints_bp.route('/getstats')
def get_stats():
"""Return serialized node statistics."""
@ -132,24 +87,6 @@ class PrivateEndpoints:
def show_uptime():
return Response(str(client_api.getUptime()))
@private_endpoints_bp.route('/getActivePubkey')
def get_active_pubkey():
return Response(pub_key)
@private_endpoints_bp.route('/getHumanReadable')
def get_human_readable_default():
return Response(mnemonickeys.get_human_readable_ID())
@private_endpoints_bp.route('/getHumanReadable/<name>')
def get_human_readable(name):
name = unpaddedbase32.repad(bytesconverter.str_to_bytes(name))
return Response(mnemonickeys.get_human_readable_ID(name))
@private_endpoints_bp.route('/getBase32FromHumanReadable/<words>')
def get_base32_from_human_readable(words):
return Response(
bytesconverter.bytes_to_str(mnemonickeys.get_base32(words)))
@private_endpoints_bp.route('/setonboarding', methods=['POST'])
def set_onboarding():
return Response(
@ -159,42 +96,6 @@ class PrivateEndpoints:
def get_os_system():
return Response(platform.system().lower())
@private_endpoints_bp.route('/gettorsocks')
def get_tor_socks():
while True:
try:
return Response(
str(
g.too_many.get_by_string(
'NetController').socksPort))
except KeyError:
sleep(0.1)
@private_endpoints_bp.route('/torready')
def is_tor_ready():
"""If Tor is starting up, the web UI is not ready to be used."""
try:
return Response(
str(g.too_many.get_by_string('NetController').readyState).lower())
except KeyError:
return Response("false")
@private_endpoints_bp.route('/gettoraddress')
def get_tor_address():
"""Return public Tor v3 Onion address for this node"""
if not config.get('general.security_level', 0) == 0:
abort(404)
return Response(get_tor()[0])
@private_endpoints_bp.route('/getgeneratingblocks')
def get_generating_blocks() -> Response:
return Response(
','.join(
g.too_many.get_by_string('DeadSimpleKV').get(
'generating_blocks'
))
)
@private_endpoints_bp.route('/getblockstoupload')
def get_blocks_to_upload() -> Response:
return Response(

View File

@ -1,65 +0,0 @@
'''
Onionr - Private P2P Communication
Create blocks with the client api server
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
from flask import Blueprint, Response, abort
from onionrblocks import onionrblockapi
from .. import apiutils
from onionrutils import stringvalidators
from coredb import blockmetadb
client_get_block = apiutils.GetBlockData()
client_get_blocks = Blueprint('miscclient', __name__)
@client_get_blocks.route('/getblocksbytype/<name>')
def get_blocks_by_type_endpoint(name):
blocks = blockmetadb.get_blocks_by_type(name)
return Response(','.join(blocks))
@client_get_blocks.route('/getblockbody/<name>')
def getBlockBodyData(name):
resp = ''
if stringvalidators.validate_hash(name):
try:
resp = onionrblockapi.Block(name, decrypt=True).bcontent
except TypeError:
pass
else:
abort(404)
return Response(resp)
@client_get_blocks.route('/getblockdata/<name>')
def getData(name):
resp = ""
if stringvalidators.validate_hash(name):
if name in blockmetadb.get_block_list():
try:
resp = client_get_block.get_block_data(name, decrypt=True)
except ValueError:
pass
else:
abort(404)
else:
abort(404)
return Response(resp)
@client_get_blocks.route('/getblockheader/<name>')
def getBlockHeader(name):
resp = client_get_block.get_block_data(name, decrypt=True, headerOnly=True)
return Response(resp)

View File

@ -1,27 +0,0 @@
from flask import Blueprint
from flask import Response
import unpaddedbase32
from coredb import blockmetadb
import onionrblocks
from etc import onionrvalues
import config
from onionrutils import bytesconverter
bp = Blueprint('motd', __name__)
signer = config.get("motd.motd_key", onionrvalues.MOTD_SIGN_KEY)
@bp.route('/getmotd')
def get_motd()->Response:
motds = blockmetadb.get_blocks_by_type("motd")
newest_time = 0
message = "No MOTD currently present."
for x in motds:
bl = onionrblocks.onionrblockapi.Block(x)
if not bl.verifySig() or bl.signer != bytesconverter.bytes_to_str(unpaddedbase32.repad(bytesconverter.str_to_bytes(signer))): continue
if not bl.isSigner(signer): continue
if bl.claimedTime > newest_time:
newest_time = bl.claimedTime
message = bl.bcontent
return Response(message, headers={"Content-Type": "text/plain"})

View File

@ -1,6 +0,0 @@
from . import announce, upload, getblocks, endpoints
announce = announce.handle_announce # endpoint handler for accepting peer announcements
upload = upload.accept_upload # endpoint handler for accepting public uploads
public_block_list = getblocks.get_public_block_list # endpoint handler for getting block lists
public_get_block_data = getblocks.get_block_data # endpoint handler for responding to peers requests for block data

View File

@ -1,62 +0,0 @@
"""Onionr - Private P2P Communication.
Handle announcements to the public API server
"""
from flask import Response, g
import deadsimplekv
import logger
from etc import onionrvalues
from onionrutils import stringvalidators, bytesconverter
import filepaths
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def handle_announce(request):
"""accept announcement posts, validating POW
clientAPI should be an instance of the clientAPI server running,
request is a instance of a flask request
"""
resp = 'failure'
newNode = ''
try:
newNode = request.form['node'].encode()
except KeyError:
logger.warn('No node specified for upload')
else:
newNode = bytesconverter.bytes_to_str(newNode)
announce_queue = deadsimplekv.DeadSimpleKV(filepaths.announce_cache)
announce_queue_list = announce_queue.get('new_peers')
if announce_queue_list is None:
announce_queue_list = []
else:
if len(announce_queue_list) >= onionrvalues.MAX_NEW_PEER_QUEUE:
newNode = ''
if stringvalidators.validate_transport(newNode) and \
newNode not in announce_queue_list:
g.shared_state.get(
deadsimplekv.DeadSimpleKV).get('newPeers').append(newNode)
announce_queue.put('new_peers',
announce_queue_list.append(newNode))
announce_queue.flush()
resp = 'Success'
resp = Response(resp)
if resp == 'failure':
return resp, 406
return resp

View File

@ -1,91 +0,0 @@
"""Onionr - Private P2P Communication.
Misc public API endpoints too small to need their own file
and that need access to the public api inst
"""
from flask import Response, Blueprint, request, send_from_directory, abort, g
from . import getblocks, upload, announce
from coredb import keydb
import config
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
class PublicEndpoints:
def __init__(self, public_api):
public_endpoints_bp = Blueprint('publicendpoints', __name__)
self.public_endpoints_bp = public_endpoints_bp
@public_endpoints_bp.route('/')
def banner():
# Display info to people who visit a node address in their browser
try:
with open('../static-data/index.html', 'r') as html:
resp = Response(html.read(), mimetype='text/html')
except FileNotFoundError:
resp = Response("")
return resp
@public_endpoints_bp.route('/getblocklist')
def get_block_list():
"""Get a list of blocks, optionally filtered by epoch time stamp,
excluding those hidden"""
return getblocks.get_public_block_list(public_api, request)
@public_endpoints_bp.route('/getdata/<name>')
def get_block_data(name):
# Share data for a block if we have it and it isn't hidden
return getblocks.get_block_data(public_api, name)
@public_endpoints_bp.route('/www/<path:path>')
def www_public(path):
# A way to share files directly over your .onion
if not config.get("www.public.run", True):
abort(403)
return send_from_directory(
config.get('www.public.path', 'static-data/www/public/'), path)
@public_endpoints_bp.route('/plaintext')
def plaintext_enabled_endpoint():
return Response(str(config.get("general.store_plaintext_blocks", True)).lower())
@public_endpoints_bp.route('/ping')
def ping():
# Endpoint to test if nodes are up
return Response("pong!")
@public_endpoints_bp.route('/pex')
def peer_exchange():
response = ','.join(keydb.listkeys.list_adders(recent=3600))
if len(response) == 0:
response = ''
return Response(response)
@public_endpoints_bp.route('/announce', methods=['post'])
def accept_announce():
"""Accept announcements with pow token to prevent spam"""
g.shared_state = public_api._too_many
resp = announce.handle_announce(request)
return resp
@public_endpoints_bp.route('/upload', methods=['post'])
def upload_endpoint():
"""Accept file uploads.
In the future this will be done more often than on creation
to speed up block sync
"""
return upload.accept_upload(request)

View File

@ -1,73 +0,0 @@
"""Onionr - Private P2P Communication.
Public endpoints to get block data and lists
"""
from flask import Response, abort
import config
from onionrutils import bytesconverter, stringvalidators
from coredb import blockmetadb
from utils import reconstructhash
from onionrblocks import BlockList
from onionrblocks.onionrblockapi import Block
from .. import apiutils
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def get_public_block_list(public_API, request):
# Provide a list of our blocks, with a date offset
date_adjust = request.args.get('date')
type_filter = request.args.get('type')
b_list = blockmetadb.get_block_list(date_rec=date_adjust)
share_list = ''
if config.get('general.hide_created_blocks', True):
for b in public_API.hideBlocks:
if b in b_list:
# Don't share blocks we created if they haven't been *uploaded* yet, makes it harder to find who created a block
b_list.remove(b)
for b in b_list:
if type_filter:
if Block(b, decrypt=False).getType() != type_filter:
continue
share_list += '%s\n' % (reconstructhash.deconstruct_hash(b),)
return Response(share_list)
def get_block_data(public_API, b_hash):
"""return block data by hash unless we are hiding it"""
resp = ''
b_hash = reconstructhash.reconstruct_hash(b_hash)
if stringvalidators.validate_hash(b_hash):
if not config.get('general.hide_created_blocks', True) \
or b_hash not in public_API.hideBlocks:
if b_hash in public_API._too_many.get(BlockList).get():
block = apiutils.GetBlockData().get_block_data(
b_hash, raw=True, decrypt=False)
try:
# Encode in case data is binary
block = block.encode('utf-8')
except AttributeError:
# 404 if no block data
if not block:
abort(404)
if not len(block):
abort(404)
resp = block
if len(resp) == 0:
abort(404)
resp = ""
# Has to be octet stream, otherwise binary data fails hash check
return Response(resp, mimetype='application/octet-stream')

View File

@ -1,94 +0,0 @@
"""Onionr - Private P2P Communication.
Accept block uploads to the public API server
"""
import sys
from gevent import spawn
from flask import Response
from flask import abort
from flask import g
from onionrutils import localcommand
from onionrblocks import blockimporter
import onionrexceptions
import logger
import config
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def accept_upload(request):
"""Accept uploaded blocks to our public Onionr protocol API server"""
resp = 'failure'
data = request.get_data()
data_size = sys.getsizeof(data)
b_hash = None
if data_size < 30:
resp = 'size'
elif data_size < 100000000:
try:
b_hash = blockimporter.import_block_from_data(data)
if b_hash:
# Upload mixing is where a node will hide and reupload a block
# to act like it is also a creator
# This adds deniability but is very slow
if g.too_many.get_by_string(
"DeadSimpleKV").get('onlinePeers') and \
config.get('general.upload_mixing', False):
spawn(
localcommand.local_command,
'/daemon-event/upload_event',
post=True,
is_json=True,
post_data={'block': b_hash}
).get(timeout=10)
resp = 'success'
else:
resp = 'failure'
logger.warn(
f'Error encountered importing uploaded block {b_hash}')
except onionrexceptions.BlacklistedBlock:
logger.debug('uploaded block is blacklisted')
resp = 'failure'
except onionrexceptions.InvalidProof:
resp = 'proof'
except onionrexceptions.DataExists:
resp = 'exists'
except onionrexceptions.PlaintextNotSupported:
logger.debug(f"attempted plaintext upload to us: {b_hash}")
resp = 'failure'
except onionrexceptions.InvalidMetadata:
logger.debug(
f'uploaded block {b_hash} has invalid metadata')
resp = 'failure'
if resp == 'failure':
abort(400)
elif resp == 'size':
resp = Response(resp, 400)
logger.warn(
f'Error importing uploaded block, invalid size {b_hash}')
elif resp == 'proof':
resp = Response(resp, 400)
if b_hash:
logger.warn(
f'Error importing uploaded block, invalid proof {b_hash}')
else:
logger.warn(
'Error importing uploaded block, invalid proof')
else:
resp = Response(resp)
return resp

View File

@ -1,94 +0,0 @@
"""Onionr - Private P2P Communication.
view and interact with onionr sites
"""
import base64
import binascii
import mimetypes
import unpaddedbase32
from flask import Blueprint, Response, request, abort
from onionrblocks import onionrblockapi
import onionrexceptions
from onionrutils import stringvalidators
from onionrutils import mnemonickeys
from . import sitefiles
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
site_api = Blueprint('siteapi', __name__)
@site_api.route('/site/<name>/', endpoint='site')
def site(name: str)->Response:
"""Accept a site 'name', if pubkey then show multi-page site, if hash show single page site"""
resp: str = 'Not Found'
mime_type = 'text/html'
# If necessary convert the name to base32 from mnemonic
if mnemonickeys.DELIMITER in name:
name = mnemonickeys.get_base32(name)
# Now make sure the key is regardless a valid base32 format ed25519 key (readding padding if necessary)
if stringvalidators.validate_pub_key(name):
name = unpaddedbase32.repad(name)
resp = sitefiles.get_file(name, 'index.html')
elif stringvalidators.validate_hash(name):
try:
resp = onionrblockapi.Block(name).bcontent
except onionrexceptions.NoDataAvailable:
abort(404)
except TypeError:
pass
try:
resp = base64.b64decode(resp)
except binascii.Error:
pass
if resp == 'Not Found' or not resp:
abort(404)
return Response(resp)
@site_api.route('/site/<name>/<path:file>', endpoint='siteFile')
def site_file(name: str, file: str)->Response:
"""Accept a site 'name', if pubkey then show multi-page site, if hash show single page site"""
resp: str = 'Not Found'
mime_type = mimetypes.MimeTypes().guess_type(file)[0]
# If necessary convert the name to base32 from mnemonic
if mnemonickeys.DELIMITER in name:
name = mnemonickeys.get_base32(name)
# Now make sure the key is regardless a valid base32 format ed25519 key (readding padding if necessary)
if stringvalidators.validate_pub_key(name):
name = unpaddedbase32.repad(name)
resp = sitefiles.get_file(name, file)
elif stringvalidators.validate_hash(name):
try:
resp = onionrblockapi.Block(name).bcontent
except onionrexceptions.NoDataAvailable:
abort(404)
except TypeError:
pass
try:
resp = base64.b64decode(resp)
except binascii.Error:
pass
if resp == 'Not Found' or not resp:
abort(404)
return Response(resp, mimetype=mime_type)

Some files were not shown because too many files have changed in this diff Show More