uploads now have sessions

added python info to version command
work on fixing uploads
2019-09-19 18:34:16 -05:00 · 2019-09-19 15:16:31 -05:00 · 2019-09-18 15:56:29 -05:00
558 changed files with 25167 additions and 8875 deletions
--- a/.dockerignore
+++ b/.dockerignore
@ -1,6 +1,4 @@
 onionr/data/**/*
 onionr/data
+RUN-WINDOWS.bat
 MY-RUN.sh
-Dockerfile
-.dockerignore
-.git
--- a/.env
+++ b/.env
@ -1 +0,0 @@
-PYTHONPATH=./venv/bin/python310:./src/
--- a/.github/stale.yml
+++ b/.github/stale.yml
@ -1,7 +0,0 @@
-# Number of days of inactivity before an issue becomes stale
-daysUntilStale: 0
-# Number of days of inactivity before a stale issue is closed
-daysUntilClose: 0
-# Label to use when marking an issue as stale
-staleLabel: wontfix
-closeComment: false
--- a/.github/workflows/greetings.yml
+++ b/.github/workflows/greetings.yml
@ -1,13 +0,0 @@
-name: Greetings
-
-on: [pull_request, issues]
-
-jobs:
-  greeting:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/first-interaction@v1
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        issue-message: 'Message that will be displayed on users'' first issue'
-        pr-message: 'Hi, please make the PR at https://gitlab.com/beardog/Onionr'' first pr'
--- a/.gitignore
+++ b/.gitignore
@ -1,44 +1,39 @@
 __pycache__/
-src/data/config.ini
-src/data/*.db
-src/data-old/*
-src/data*
-src/tor
-src/tor.exe
-src/testdata
-src/*.pyc
-src/*.log
-src/data/hs/hostname
-src/data/*
-src/data-backup/*
+onionr/data/config.ini
+onionr/data/*.db
+onionr/data-old/*
+onionr/data*
+onionr/tor
+onionr/tor.exe
+onionr/testdata
+onionr/*.pyc
+onionr/*.log
+onionr/data/hs/hostname
+onionr/data/*
+onionr/data-backup/*
+onionr/gnupg/*
 run.sh
-src/.onionr-lock
-daemon-true.txt
-.vscode/tags
-.vscode/settings.json
-
-venv*
-src/fs*
-src/tmp/*
+onionr/data-encrypted.dat
+onionr/.onionr-lock
+core
+.vscode/*
+venv/*
+onionr/fs*
+onionr/tmp/*
 testdata/*
 *.dll
 *.exe

-.mypy_cache/
-
-dist/*
-
 # log files
 output.log
 *.log
-src/output.log
-src/*.log
-src/data/output.log
-src/data/*.log
+onionr/output.log
+onionr/*.log
+onionr/data/output.log
+onionr/data/*.log

 # package files
 onionr-*.pkg.tar.gz
 pkg/
+src/
 spawnnodes.py
-
-static-data/tor-node-list.dat
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -1,11 +1,8 @@
 test:
-  image: python
+  image: ubuntu:disco
  script:
  - apt-get update -qy
-  - apt-get install -y tor
-  - apt-get install -y firefox-esr
+  - apt-get install -y python3-pip tor
  - pip3 install --require-hashes -r requirements.txt
-  - pip3 install --require-hashes -r requirements-dev.txt
-  - pip3 install --require-hashes -r requirements-notifications.txt
  - python3 --version
  - make test
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@ -1,23 +0,0 @@
-{
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "name": "Onionr: init",
-            "type": "python",
-            "request": "launch",
-            "program": "${workspaceFolder}/src/__init__.py",
-            "cwd": "${workspaceFolder}/src/",
-            "args" : ["start"]
-        },
-        {
-            "name": "Python: Current File",
-            "type": "python",
-            "request": "launch",
-            "program": "${file}",
-            "cwd": "${workspaceFolder}/src/"
-        }
-    ]
-}
--- a/.vscode/tasks.json
+++ b/.vscode/tasks.json
@ -1,58 +0,0 @@
-{
-    // See https://go.microsoft.com/fwlink/?LinkId=733558
-    // for the documentation about the tasks.json format
-    "version": "2.0.0",
-    "tasks": [
-      {
-        "label": "Make new test",
-        "type": "shell",
-        "command": "cp tests/test_template.py tests/new-test.py",
-        "group": "test",
-        "presentation": {
-          "reveal": "always",
-          "panel": "new"
-        }
-      },
-      {
-        "label": "Run tests",
-        "type": "shell",
-        "command": "source venv/bin/activate; make test",
-        "group": "test",
-        "presentation": {
-          "reveal": "always",
-          "panel": "new"
-        }
-      },
-      {
-        "label": "Run test by name",
-        "type": "shell",
-        "command": "source venv/bin/activate; scripts/run-unit-test-by-name.py",
-        "group": "test",
-        "presentation": {
-          "reveal": "always",
-          "panel": "new"
-        }
-      },
-      {
-        "label": "Enable dev config",
-        "type": "process",
-        "command": "scripts/enable-dev-config.py",
-        "group": "dev",
-        "problemMatcher": []
-      },
-      {
-        "label": "Disable dev config",
-        "type": "process",
-        "command": "scripts/disable-dev-config.py",
-        "group": "dev",
-        "problemMatcher": []
-      },
-      {
-        "label": "Format default config",
-        "type": "process",
-        "command": "scripts/pretty-default-config.py",
-        "group": "dev",
-        "problemMatcher": []
-      }
-    ]
-  }
--- a/AUTHORS.MD
+++ b/AUTHORS.MD
@ -0,0 +1,12 @@
+# Incomplete List of Contributors
+
+Onionr is created by a team of hard working volunteers.
+
+In no order of importance, these people make Onionr happen:
+
+* [Beardog (Kevin Froman)](https://www.chaoswebs.net/) - Project founder, owner and core developer
+* [InvisaMage](https://invisamage.com/) - Web UI Bulma design
+* [Arinerron](https://arinerron.com/) - Logger and config modules, testing and other contributions
+* [Anhar Ismail](https://github.com/anharismail) - Created Onionr's logo
+
+ Other contributors and testers
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -1,48 +0,0 @@
-# Changelog
-
-All notable changes to this project will be documented in this file.
-
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
-and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-
-## [8.0.1] - 2020-12-22
-
-* Fix subprocess in 3.9x with dumb hack
-* Dependency bumps
-
-## [8.0.0] - 2020-12-04
-
-* Decrease PoW until better implementation is made
-
-
-## [7.2.0] - 2020-12-03
-
-* Purge blocks not meeting current pow on startup
-* Check block POW before LAN sync
-* WSL fixes
-
-## [7.1.0] - 2020-11-23
-
-* Check for ownership of existing dirs in createdirs, this prevents the rare edge case where a user might use a home directory in a location an attacker could write (allowing arbitrary code execution via plugins). This was already partially mitigated by the chmod of the home directory in any case, but this further fixes the issue.
-
-## [7.0.0] - 2020-11-22
-
-* Removed communicator timers
-* Removed direct connections and chat (these will be either plugins or separate programs/processes in the future)
-
-
-## [5.1.0] - 2020-09-07
-
-* Moved plugin web files to be in the plugin folder to reduce staticfiles blueprint coupling
-* Added basic sidebar on index page
-* Many bug fixes
-
-
-## [5.0.1] - 2020-08-08
-
-* bumped deadsimplekv to 0.3.2
-* bumped urllib3 to 1.25.10
-
-## [5.0.0] - 2020-07-23
-
- Removed single-process POW support (was only needed on Windows)
--- a/35
+++ b/35
@ -1,31 +1,28 @@
-FROM python:3.10
-EXPOSE 8080
+FROM ubuntu:disco

-USER root
-
-RUN mkdir /app
-WORKDIR /app
-
-ENV ONIONR_DOCKER=true
+#Base settings
+ENV HOME /root

 #Install needed packages
-RUN apt-get update && apt-get install -y tor locales
+RUN apt update && apt install -y python3 python3-dev python3-pip tor locales nano sqlite3

 RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
    locale-gen
-ENV LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8
+ENV LANG en_US.UTF-8  
+ENV LANGUAGE en_US:en  
+ENV LC_ALL en_US.UTF-8  

-ADD ./requirements-x86-all-plugins.txt /app/requirements.txt
+WORKDIR /srv/
+ADD ./requirements.txt /srv/requirements.txt
 RUN pip3 install --require-hashes -r requirements.txt

+WORKDIR /root/
 #Add Onionr source
-COPY . /app/
+COPY . /root/
+VOLUME /root/data/

-VOLUME /app/data/
+#Set upstart command
+CMD bash

-#Default to running as nonprivileged user
-RUN chmod g=u -R /app
-USER 1000
-ENV HOME=/app
-
-CMD ["bash", "./onionr.sh", "start"]
+#Expose ports
+EXPOSE 8080
--- a/ISSUE_TEMPLATE.md
+++ b/ISSUE_TEMPLATE.md
@ -5,9 +5,8 @@
 # Steps to Reproduce

 # Version Information
-
-* Onionr: 
-* OS: 
-* Python: 
-* Tor: 
-* CPU:
+Onionr: 
+OS: 
+Python: 
+Tor: 
+I2P: 
--- a/2
+++ b/2
@ -30,7 +30,7 @@ soft-reset:

 reset:
 	@echo "Hard-resetting Onionr..."
-	rm -rf $(ONIONR_HOME)/ | true > /dev/null 2>&1
+	rm -rf onionr/$(ONIONR_HOME)/ | true > /dev/null 2>&1
 	cd onionr/static-data/www/ui/; rm -rf ./dist; python compile.py
 	#@./onionr.sh version | grep -v "Failed" --color=always

--- a/README.md
+++ b/README.md
@ -5,107 +5,107 @@
 </p>

 <p align="center">
-    Privacy Respecting Communication Network 📡
-</p>
-<p align="center">
-    WIP anonymous social platform, mail, file sharing and marketplace
+    Private P2P Communication Network 📡
 </p>

-<img src="https://img.shields.io/badge/License-aGPLv3-yellow"> <img src='https://img.shields.io/badge/python%20version%20%F0%9F%90%8D-3.10+-blue'>
+<img src='https://img.shields.io/github/license/beardog108/onionr'> <img src='https://gitlab.com/beardog/Onionr/badges/master/build.svg'> <img src='https://img.shields.io/badge/docker%20%F0%9F%90%8B-supported-success'> <img src='https://img.shields.io/badge/python%20version%20%F0%9F%90%8D-3.6+-blue'> <img src='https://img.shields.io/github/commit-activity/m/beardog108/onionr'>

-<a href='https://twitter.com/onionrnet'><img src='https://img.shields.io/twitter/follow/onionrnet?style=social'></a> - [Discord](https://discord.gg/DVF2bEAzrt) - Matrix: #onionr:amorgan.xyz
+<img src='https://onionr.net/block-count.svg' alt='current stored block count'>
+
+<a href='https://www.reddit.com/r/onionr'><img src = 'https://img.shields.io/reddit/subreddit-subscribers/onionr?style=social'></a> <a href='https://twitter.com/onionrnet'><img src='https://img.shields.io/twitter/follow/onionrnet?style=social'></a>

 |                                      |                            |                            |
 | -----------                          | -----------                | -----------                |
 | [Install](#install-and-run-on-linux) | [Features](#main-features) | [Screenshots](#screenshots)|
-| [Docs](#documentation)               | [Get involved](#help-out)  | [Onionr.net](https://onionr.net/)/[.onion](http://onionrbak72t5zhbzuey2fdkpczlvhowgcpqc6uoyrd3uxztzxwz5cyd.onion/)                  |
+| [Docs](#documentation)               | [Get involved](#help-out)  | [Onionr.net](https://onionr.net/)/[.onion](http://onionr.onionkvc5ibm37bmxwr56bdxcdnb6w3wm4bdghh5qo6f6za7gn7styid.onion/)                  |


---
+<hr>

-**The main repository for this software is at https://gitlab.com/beardog/onionr/**
+**The main repository for this software is at https://GitLab.com/beardog/Onionr/**

-***Note that this README reflects the state of the rewrite, and not the original alpha network***
+**The [GitHub repository](https://github.com/beardog108/onionr/) is a mirror, do not submit PRs or issues there.**

-Mirrors [Gitea](https://git.voidnet.tech/kev/onionr)
+# About

+Onionr ("Onion Relay") is a decentralized, peer-to-peer communication network, designed to be anonymous and resistant to (meta)data analysis, spam, and corruption.

-Onionr ("Onion Relay") is a decentralized/distributed peer-to-peer communication network, designed to be anonymous and resistant to (meta)data analysis, spam, and corruption.
-
-Onionr gives the individual the ability to speak freely, without fear of surveillance and censorship.
-
---
-
-# Onionr internals
-
-## Blocks
-
-At the core, Onionr is an anonymous Distributed Hash Table (DHT) syncing prepackaged blocks using a simple Gossip protocol with Dandelion++ as an overlay network on top of Tor and I2P.
-
-Onionr stores data in independent packages referred to as 'blocks'. The blocks are distributed to all nodes, but are not required to be stored. Blocks and user IDs cannot be easily proven to have been created by a particular user. Even if Dandelion++ is defeated and there is enough evidence to believe that a specific node is linked to a block's creation, nodes still operate behind Tor or I2P and as such cannot be trivially unmasked. Anonymity is achieved by a stateless network, with no given indication of what node a block originates from. In fact, since one is not required to participate in routing or storage to insert a message, blocks often do not originate from any identifiable node, similar to how Bitcoin transactions do not necessarily originate from a wallet directly associated with a node.
-
-## Onionr Gossip
-
-Onionr works via epidemic/gossip style routing, with message delivery taking roughly log<sub>C</sub>(N) cycles where C is the number of nodes to send a message to each cycle and N is the number of connected nodes. So a network of 100 million nodes can deliver messages in a few minutes even with high packet loss and malfunctioning nodes.
-
-Through Dandelion++ message forwarding and key privacy, it is intended to be nigh impossible to discover the identity of a message creator or recipient. Via long-term traffic analysis, a well funded adversary may discover the most probable node(s) to be creating a set of related blocks, however doing so would only lead them to a node behind Tor. As the first node that a block appears on is almost always not the creator of the block, there is plausible deniability regarding the true creator of the block.
+Onionr stores data in independent packages referred to as 'blocks'. The blocks are synced to all other nodes in the network. Blocks and user IDs cannot be easily proven to have been created by a particular user. Even if there is enough evidence to believe that a specific user created a block, nodes still operate behind Tor or I2P and as such cannot be trivially unmasked.

 Users are identified by ed25519/curve25519 public keys, which can be used to sign blocks or send encrypted data.

 Onionr can be used for mail, as a social network, instant messenger, file sharing software, or for encrypted group discussion.

-Since Onionr is technically just a data format, any routing scheme can  technically be used to pass messages.
+The whitepaper (subject to change prior to alpha release) is available [here](docs/whitepaper.md).

-The whitepaper is available [here](docs/whitepaper.md).
+![Tor stinks slide image](docs/tor-stinks-02.png)

---
-
-# Main Features
+## Main Features

 * [X] 🌐 Fully p2p/decentralized, no trackers or other single points of failure
+* [X] 🔒 End to end encryption of user data
+* [X] 📢 Optional non-encrypted blocks, useful for blog posts or public file sharing
+* [X] 💻 Easy HTTP API for integration to websites
 * [X] 🕵️ Metadata analysis resistance and anonymity
 * [X] 📡 Transport agnosticism (no internet required)

+Onionr ships with various application plugins ready for use out of the box:

-# Roadmap
+Currently usable:

-See [ROADMAP.md](ROADMAP.md)
+* Mail
+* Public anonymous chat/message board
+* Simple webpage hosting - Will be greatly extended
+* File sharing (Work in progress)
+
+Not yet usable:
+
+* Instant messaging
+
+**Onionr API and functionality is subject to non-backwards compatible change during pre-alpha development**
+
+# Screenshots
+
+<img alt='Node statistics page screenshot' src='docs/onionr-1.png' width=600>
+
+Home screen
+
+<img alt='Friend/contact manager screenshot' src='docs/onionr-2.png' width=600>
+
+Friend/contact manager
+
+<img alt='Encrypted, metadata-masking mail application screenshot' src='docs/onionr-3.png' width=600>
+
+Encrypted, metadata-masking mail application. One of the first distributed mail systems to have basic forward secrecy.

 # Documentation

 More docs coming soon.

+* [Block specification](docs/specs/block-spec.md)
+* [HTTP API](docs/http-api.md)

 # Install and Run on Linux

-The following applies to Ubuntu Bionic. Other distributions may have different package or command names.
-
-Master may be unstable, you should use the latest release tag. (checkout via git: `$ git checkout release-latest`)
+The following applies to Ubuntu Bionic. Other distros may have different package or command names.

 `$ sudo apt install python3-pip python3-dev tor`

-* Have python3.10, python3-pip, Tor (daemon, not browser) installed. python3-dev is recommended.
-* You may need build-essentials or the equivalent of your platform
-* Clone the git repo: `$ git clone https://gitlab.com/beardog/onionr --tags`
+* Have python3.6+, python3-pip, Tor (daemon, not browser) installed. python3-dev is recommended.
+* Clone the git repo: `$ git clone https://gitlab.com/beardog/onionr`
 * cd into install direction: `$ cd onionr/`
-* Install the Python dependencies ([virtualenv strongly recommended](https://virtualenv.pypa.io/en/stable/userguide/)): `$ pip3 install --require-hashes -r requirements-x86-all-plugins.txt`
+* Install the Python dependencies ([virtualenv strongly recommended](https://virtualenv.pypa.io/en/stable/userguide/)): `$ pip3 install --require-hashes -r requirements.txt`
+* (Optional): Install desktop notification dependencies: `$ pip3 install --require-hashes -r requirements-notifications.txt`

-Require-hashes is suggested for supply-chain security but is optional. The hashes are not correct for ARM machines. If you are just running a node or want a bare-bones install you can use requirements-base-x86.txt and selectively install the requirements.txt files in static-data/official-plugins/ subdirectories
+(--require-hashes is intended to prevent exploitation via compromise of PyPi/CA certificates)

 ## Run Onionr

 * Run Onionr normally:  `$ ./onionr.sh start`
 * Run Onionr in background as daemon: `$ ./start-daemon.sh`
+* Open Onionr web interface  `$ ./onionr.sh openhome`
 * Gracefully stop Onionr from CLI `$ ./onionr.sh stop`

-
-# Contact/Community
-
-* Email: onionr [ at ] voidnet.tech
-* Twitter: [@onionrnet](https://twitter.com/onionrnet)
-* Matrix: #onionr:amorgan.xyz
-* Discord: https://discord.gg/DVF2bEAzrt (Discord is bad for freedom and privacy, this is only provided for convenience)
-
 # Help out

 Everyone is welcome to contribute. Help is wanted for the following:
@ -113,41 +113,36 @@ Everyone is welcome to contribute. Help is wanted for the following:
 * Development (Get in touch first)
    * Creation of a shared lib for use from other languages and faster proof-of-work
    * Android and IOS development
-    * Mac support (testers needed)
+    * Windows and Mac support (already partially supported, testers needed)
    * Bug fixes and development of new features
 * Testing
 * Translations/localizations
 * UI/UX design
 * Running stable nodes
 * Security review/audit
-* I2P support
-
-## Watch the talk from BSidesPDX 2019
-
-<a href="https://www.youtube.com/watch?v=mrULtmSkKxg">
-<img src="docs/talk.png" alt="improving anonymous networking talk link" width="600">
-</a>
+* Automatic I2P setup

 ## Contribute money:

-Donating at least $3 gets you cool Onionr stickers. Get in touch if you want them.
+Donating at least $5 gets you cool Onionr stickers. Get in touch if you want them.

-![sticker](docs/sticker.png)
+Bitcoin: [1onion55FXzm6h8KQw3zFw2igpHcV7LPq](bitcoin:1onion55FXzm6h8KQw3zFw2igpHcV7LPq) (Contact us for a unique address or for other coins)

+Monero: 4B5BA24d1P3R5aWEpkGY5TP7buJJcn2aSGBVRQCHhpiahxeB4aWsu15XwmuTjC6VF62NApZeJGTS248RMVECP8aW73Uj2ax

-* Bitcoin: [bc1qpayme9rlpkch0qp3r79lvm5racr7t6llauwfmg](bitcoin:bc1qpayme9rlpkch0qp3r79lvm5racr7t6llauwfmg) (Contact us for a unique address or for other coins)
+USD (Card/Paypal): [Ko-Fi](https://www.ko-fi.com/beardogkf)

-* Monero: 4B5BA24d1P3R5aWEpkGY5TP7buJJcn2aSGBVRQCHhpiahxeB4aWsu15XwmuTjC6VF62NApZeJGTS248RMVECP8aW73Uj2ax
+Note: probably not tax deductible

-* USD (Card/Paypal (no account required)): [Ko-Fi](https://www.ko-fi.com/beardogkf)
+# Contact

-* Sign up for [privacy.com (refferal link)](https://privacy.com/join/FNNDF) to protect your personal information when contributing or shopping elsewhere, we both get $5 USD.
+Email: beardog [ at ] mailbox.org

-Note: not tax deductible
+Onionr Mail: TRH763JURNY47QPBTTQ4LLPYCYQK6Q5YA33R6GANKZK5C5DKCIGQ

 # Security

-Onionr is alpha software. This means it is unstable, probably insecure, and experimental.
+Onionr is pre-alpha. This means it is unstable, probably insecure, and experimental.

 No matter how good Onionr and other software gets, there will always be ways for clever or well-funded adversaries to break your security.

@ -165,6 +160,6 @@ Tor is a trademark for the Tor Project. We do not own it.

 ## Onionr Logo

-The Onionr logo was created by [Anhar Ismail](https://github.com/anharismail) under the [Creative Commons Attribution 4.0 International License](https://creativecommons.org/licenses/by/4.0/).
+The Onionr logo was created by [Anhar Ismail](https://github.com/anharismail) under the [Creative Commons Attribution 4.0 International License](https://creativecommons.org/licenses/by/4.0/). 

 If you modify and redistribute our code ("forking"), please use a different logo and project name to avoid confusion. Please do not use the project name or logo in a way that makes it seem like we endorse you without our permission.
--- a/ROADMAP.md
+++ b/ROADMAP.md
@ -1,46 +0,0 @@
-# Onionr Roadmap
-
-
-## Taraxacum Release (9.0)
-
-
-* [ X ] Implement new block format with verifiable delay function
-
-* [ X ] Implement overhauled gossip system with dandelion++
-
-    The new system is separated from the underlying networks used and makes it much easier to implement new transport methods. Dandelion++ is also better than the old block mixing we did.
-
-
-## Misc
-
-* [  ] Spruce up documentation
-* [  ] Restore LAN transport
-* [  ] Restore webUI as a plugin
-* [  ] Restore static site/file sharing plugin
-* [  ] Restore/reimplement mail plugin
-* [  ] Restore/reimplement friends plugin
-* [  ] Refresh test suite
-* [  ] Revamped key/encrypted messaging (encrypted blocks)
-
-
-
-## Web of trust release (~10.0)
-
-To facilitate the below market plugin/application, Onionr will need a web of trust system.
-
-* [ ] Web of trust plugin or module
-
-
-## Market Plugin Release (~10.1)
-
-The Onionr team believes the Monero community is currently lacking a good p2p market, and as such we hope to build a solution using Onionr as a transport. This may be a separate project and as opposed to a plugin.
-
-
-* A new marketplace plugin will be developed with the following *tenative* features:
-
-* [ ] Account management
-* [ ] Monero management
-* [ ] Store front discovery and search
-* [ ] Product listing search
-* [ ] User reviews
-
--- a/base-requirements.in
+++ b/base-requirements.in
@ -1,8 +0,0 @@
-PyNaCl==1.5.0
-psutil==5.9.1
-filenuke==0.0.0
-ujson==5.4.0
-cffi==1.15.1
-onionrblocks==7.0.0
-ordered-set==4.1.0
-json-rpc==1.13.0
--- a/docs/README.md
+++ b/docs/README.md
@ -1,6 +1,4 @@
-<img src="onionr-logo.png" width="200px">
-
-# Documentation
+# Onionr Documentation

 The Onionr [whitepaper](whitepaper.md) is the best place to start both for users and developers.

@ -18,4 +16,4 @@ The Onionr [whitepaper](whitepaper.md) is the best place to start both for users
 * [Technical overview](dev/overview.md)
 * [Project layout](dev/layout.md)
 * [Plugin development guide](dev/plugins.md)
-* [Auto generated API docs (HTML)](html/onionr/index.html)
+* [Testing](dev/testing.md)
--- a/docs/basic-onionr-user-guide.pdf
+++ b/docs/basic-onionr-user-guide.pdf
--- a/docs/dev/daemon-events.md
+++ b/docs/dev/daemon-events.md
@ -1,9 +0,0 @@
-# DaemonEvents
-
-For things that need to be processed by the daemon
-
-Observer pattern
-
-Register listeners dynamically per event
-
-Spawn new greenlets
--- a/docs/dev/error-codes.md
+++ b/docs/dev/error-codes.md
@ -1,4 +0,0 @@
-# Exit codes
-
-10: run with no command
-3: command does not exist
--- a/docs/dev/generating-requirements.txt.md
+++ b/docs/dev/generating-requirements.txt.md
@ -1,19 +0,0 @@
-# Generating requirements.txt
-
-To generate a requirements.txt file, install pip-tools from pip
-
-Onionr requirements files should have hashes to prevent backdooring by the pypi server.
-
-Put your package versions in requirements.in like normal. Child dependencies are usually not necessary:
-
-```
-requests==0.1.1
-flask==0.1.1
-```
-
-Then generate the requirements.txt:
-
-`$ pip-compile requirements.in --generate-hashes -o requirements.txt`
-
-
-Your requirements.txt will have hash-pinned requirements of all dependencies and child dependencies.
--- a/docs/dev/http-api.md
+++ b/docs/dev/http-api.md
@ -0,0 +1,118 @@
+# Onionr HTTP API
+
+All HTTP interfaces in the Onionr reference client use the [Flask](http://flask.pocoo.org/) web framework with the [gevent](http://www.gevent.org/) WSGI server.
+
+## Client & Public difference
+
+The client API server is a locked down interface intended for authenticated local communication. 
+
+The public API server is available only remotely from Tor & I2P. It is the interface in which peers use to communicate with one another.
+
+# Client API
+
+Please note: endpoints that simply provide static web app files are not documented here.
+
+* /serviceactive/pubkey
+    - Methods: GET
+    - Returns true or false based on if a given public key has an active direct connection service.
+* /queueResponseAdd/key (DEPRECATED)
+    - Methods: POST
+    - Accepts form key 'data' to set queue response information from a plugin
+    - Returns success if no error occurs
+* /queueResponse/key (DEPRECATED)
+    - Methods: GET
+    - Returns the queue response for a key. Returns failure with a 404 code if a code is not set.
+* /ping
+    - Methods: GET
+    - Returns "pong!"
+* /getblocksbytype/type
+    - Methods: GET
+    - Returns a list of stored blocks by a given type
+* /getblockbody/hash
+    - Methods: GET
+    - Returns the main data section of a block
+* /getblockdata/hash
+    - Methods: GET
+    - Returns the entire data contents of a block, including metadata.
+* /getblockheader/hash
+    - Methods: GET
+    - Returns the header (metadata section) of a block.
+* /hitcount
+    - Methods: GET
+    - Return the amount of requests the public api server has received this session
+* /lastconnect
+    - Methods: GET
+    - Returns the epoch timestamp of when the last incoming connection to the public API server was logged
+* /site/hash
+    - Methods: GET
+    - Returns HTML content out of a block
+* /waitforshare/hash
+    - Methods: POST
+    - Prevents the public API server from listing or sharing a block until it has been uploaded to at least 1 peer.
+* /shutdown
+    - Methods: GET
+    - Shutdown Onionr. You should probably use /shutdownclean instead.
+* /shutdownclean
+    - Methods: GET
+    - Tells the communicator daemon to shutdown Onionr. Slower but cleaner.
+* /getstats
+    - Methods: GET
+    - Returns some JSON serialized statistics
+* /getuptime
+    - Methods: GET
+    - Returns uptime in seconds
+* /getActivePubkey
+    - Methods: GET
+    - Returns the current active public key in base32 format
+* /getHumanReadable/pubkey
+    - Methods: GET
+    - Echos the specified public key in mnemonic format
+* /insertblock
+    - Methods: POST
+    - Accepts JSON data for creating a new block. 'message' contains the block data, 'to' specifies the peer's public key to encrypt the data to, 'sign' is a boolean for signing the message.
+
+# Public API
+
+v0
+
+* /
+    - Methods: GET
+    - Returns a basic HTML informational banner describing Onionr.
+* /getblocklist
+    - Methods: GET
+    - URI Parameters:
+        - date: unix epoch timestamp for offset
+    - Returns a list of block hashes stored on the node since an offset (all blocks if no timestamp is specified)
+* /getdata/block-hash
+    - Methods: GET
+    - Returns data for a block based on a provided hash
+* /www/file-path
+    - Methods: GET
+    - Returns file data. Intended for manually sharing file data directly from an Onionr node.
+* /ping
+    - Methods: GET
+    - Returns 'pong!'
+* /pex
+    - Methods: GET
+    - Returns a list of peer addresses reached within recent time
+* /announce
+    - Methods: POST
+    - Accepts form data for 'node' (valid node address) and 'random' which is a nonce when hashed (blake2b_256) in the format `hash(peerAddress+serverAddress+nonce)`, begins with at least 5 zeros.
+    - Returns 200 with 'Success' if no error occurs. If the post is invalid, 'failure' with code 406 is returned.
+* /upload
+    - Methods: POST
+    - Accepts form data for 'block' as a 'file' upload.
+    - Returns 200 with 'success' if no error occurs. If the block cannot be accepted, 'failure' with 400 is returned.
+
+# Direct Connection API
+
+These are constant endpoints available on direct connection servers. Plugin endpoints for direct connections are not documented here.
+
+* /ping
+    - Methods: GET
+    - Returns 200 with 'pong!'
+
+* /close
+    - Methods: GET
+    - Kills the direct connection server, destroying the onion address.
+    - Returns 200 with 'goodbye'
--- a/docs/dev/running-tests.md
+++ b/docs/dev/running-tests.md
@ -1,44 +0,0 @@
-# Running Onionr tests
-
-Onionr has four types of tests:
-
-* unittests
-* integration tests
-* selenium tests (web tests)
-* runtime-tests
-
-
-## unittests
-
-Onionr uses Python's built in unittest module. These tests are located in tests/ (top level)
-
-Run all tests with `$ make test`, which will also run integration tests.
-
-Please note that one unittest tests if runtime-tests have passed recently. This is simply a forceful reminder to run those tests as well.
-
-You can also run a single unittest in a loop by using the script scripts/run-unit-test-by-name.py
-
-## integration tests
-
-These tests are pretty basic and test on stdout of Onionr commands.
-
-They are also run from `$ make test`
-
-The runtime-tests do most of the actual integration testing.
-
-## selenium tests
-
-These are browser automation tests to test if the UI is working as how it should for a user.
-
-There's only a couple and they're incomplete, so they can be ignored for now (test manually)
-
-## runtime-tests
-
-These are important. They look into the Onionr client Flask app when Onionr daemon is running and test a bunch of things.
-
-If you do it a lot you should make your own Onionr network (disable official bootstrap)
-
-You run this while the daemon is running (probably should make sure onboarding is done), with `$ onionr.sh runtime-test`
-
-It's necessary to do this before running `$ make test` for unittesting
-
--- a/docs/dev/security-mechanisms.md
+++ b/docs/dev/security-mechanisms.md
@ -1,43 +0,0 @@
-# Onionr Security Mechanisms
-
-
-## bigbrother 👁️
-
-Bigbrother is a cheeky module that uses Python3.8+ sys auditing events to log and/or block certain sensitive events.
-
-It has a little overhead, so one can disable it in config in general.security_auditing
-
-[ChaosWebs.net/blog/preventing-arbitrary-code-execution-in-python38-with-auditing.html](https://chaoswebs.net/blog/preventing-arbitrary-code-execution-in-python38-with-auditing.html)
-
-### Threat model
-
-It is intended to log bugs leaking private file system information, block+log network leaks, and block+log eval-like arbitrary code execution. It is not intended to block malicious browser scripts or malicious Python plugins. It cannot work with subprocesses that do not activate the module.
-
-It's not intended to be bulletproof by any means, but it helps.
-
-### What big brother does
-
-* Disk access checks for disk access outside. Only logs, does not block
-* Network leaks. (Non Tor/LAN) Blocks and logs
-* Arbitrary code execution: logs and blocks non-whitelisted bytecode importing/compiling and subprocesses.
-
-
-## Sybil attacks
-
-As with any decentralized network, sybil nodes could collude to spy or cause mayhem. Due to the gossip nature of Onionr, sybil nodes would have a hard time fully stopping the network. In terms of spying, they could not conclusively prove the origin of messages due to the multiple transport nature of the network and layering behind Tor/etc.
-
-## Tor configuration
-
-When managed by Onionr, Tor has a control port password that gets stored in Onionr config.
-
-Tor is also configured to reject requests to non-onion services, which helps to stop redirect based denial of service attacks.
-
-## Web security
-
-Onionr secures both it's main web APIs with anti-dns-rebinding logic, which validates the host header used in connections to it. This is to prevent exfiltration of data and side channel deanonymization.
-
-Onionr secures the client API with a token that must be passed in most requests, with the exception of static API files. This is to prevent CSRF and side channel deanonymization.
-
-Onionr binds most services to random loopback addresses to reduce all cross-site web attacks, including discovery of Onionr on a computer from a normal website. This is not supported on Mac because Mac does not support non 'typical' loopback addresses.
-
-Onionr has a strict content-security-policy, rejecting all non-localhost requests and denying inline scripts and similar insecure sources.
--- a/docs/dev/selected-papers.md
+++ b/docs/dev/selected-papers.md
@ -1,9 +0,0 @@
-# Interesting papers related to Onionr development
-
-A paper being listed here is not end-all-be-all endorsement of every detail inside.
-
-* [Epidemic Routing for Partially-Connected Ad Hoc Networks](https://web.archive.org/web/20200208074703/http://issg.cs.duke.edu/epidemic/epidemic.pdf)
-* [Freenet: A distibuted decentralized information storage and retrieval system](https://freenetproject.org/assets/papers/ddisrs.pdf)
-* [Protecting Free Expression Online with Freenet](https://freenetproject.org/assets/papers/ddisrs.pdf)
-* [Bitmessage: A Peer‐to‐Peer Message Authentication and Delivery System](https://archive.org/details/BitmessageWhitepaper/)
-* [MuON: Epidemic based Mutual Anonymity](https://web.archive.org/web/20060901153544/http://www.csl.mtu.edu/cs6461/www/Reading/MuON_ICNP2005.pdf)
--- a/docs/dev/setup.md
+++ b/docs/dev/setup.md
@ -1,72 +0,0 @@
-<h1 align="center">Onionr Developer Guide</h1>
-
-This page assumes that Onionr is already installed and normal user requirements are setup.
-
-The Onionr development environment is simple. All one really needs is a supported Python version (currently 3.7-3.8 as of writing).
-
-There are additional requirements specified in requirements-dev.txt
-
-**Developers agree to the [CoC](../../CODE_OF_CONDUCT.md) and to contribute new code under GPLv3 or later**. Developers should stick to PEP8 in most cases, and write unittests or integration tests where possible.
-
-## Developer Scripts
-
-run-onionr-node.py can be used to start a node with specific parameters
-
-Intended to be used from VSCode (but could work otherwise), there are scripts in scripts/ named enable/disable-dev-config.py.
-These make modifications to the default config for the purpose of making testing Onionr nodes easier.
-Be sure to disable it again before pushing work.
-
-There are also scripts to generate new tests.
-
-*When adjusting PoW, it will make your node not compatible with the existing network*
-
-Generally, one should disable bootstrap list usage when testing non trivial changes. This is a config option: general.use_bootstrap_list. and can be configured through enable-dev-config.py and run-onionr-node.py
-
-
-# Current state of Onionr [2021-01-14]
-
-Onionr in it's current form is functional, albeit buggy.
-
-
-## Current major components
-
-Onionr runs via two main HTTP gevent servers serving Flask apps.
-
-Dir: apiservers
-
-* 1 Parent app hosts all public API endpoints for the Tor transport.
-* 1 Parent app hosts all UI-related files and endpoints. Some commands and internal modules interact with this API as well
-* The HTTP servers have strict anti-dns-rebinding and CSRF countermeasures, so there is a script to craft requests to the UI-related API in scripts/
-* Block storage is currently handled via metadata in sqlite (mostly defunct now), and block data storage in a different database. This is in blocks/ in running Onionr daemon data directory
-* cryptography is currently handled in onionrcrypto/ except for ephemeral messages which are handled by onionr
-* Transport clients run from looping threads mostly created in communicator/__init__.py, this includes block lookups and uploading on the Tor transport
-
-## Road map
-
-There are several big ways Onionr will be improved in the next major version:
-
-* Migration to the [new modular block system](https://git.voidnet.tech/kev/onionrblocks)
-    * Probability proof of work -> verifiable delay function
-    * Friend system built on top of signing proofs (Private networks?)
-* Gossip transport improvements such as with neighbor improvements. See streamfill/ and [simple gossip](https://github.com/onion-sudo/simplegossip) for incomplete experiments
-
-* Finish removing "communicator"
-* I2P transports
-    * Gossip
-    * Torrents (patch for sha1?)
-* Modular transports
-    * Currently transports are just threads coupled together.
-    * It would be better if there was a generic way to tell any loaded transport what blocks are wanted and feed back received blocks to the database
-* Migrate to SafeDB for peers and blocks
-    * SafeDB wrapper that contacts http endpoint to store if it is running, otherwise directly open DB
-* Separate UI logic from daemon. Refactor code to
-* Improve cryptography
-    * Restore phrases or deterministic keys (generate key from password, be very careful)
-    * Change identities to be dual keys (ed25519+curve25519)
-    * Finish treasurechest
-        * Interact via [named pipes](https://en.wikipedia.org/wiki/Named_pipe)
-        * Ephemeral key management
-        * Encrypt/decrypt/sign/verify functions to keep key out of main memory
-        * PGP-like symmetric messages
-
-
--- a/docs/dev/specs/block-spec.md
+++ b/docs/dev/specs/block-spec.md
@ -1,4 +1,4 @@
-# Onionr Block Spec v2.0.0
+# Onionr Block Spec v1.1.0

 # Block Description

@ -20,13 +20,13 @@ The metadata section has the following fields. If a block contains any other fie

 ## meta

-Max byte size (when in escaped json string format): 1000
+Max byte size: 1000

-Meta is a string field which can contain arbitrary sub fields. It is intended for applications and plugins to use it for arbitrary metadata information. If the data section is encrypted or signed, the meta section also is.
+Meta is a string field which can contain arbitrary sub fields. It is intended for applications and plugins to use it for arbitrary metadata information. In the reference client, if the data section is encrypted or signed, the meta section also is.

 Common meta fields, such as 'type' are used by the reference Onionr client to describe the type of a block.

-## sig (optional)
+## sig

 Max byte size: 200

@ -34,30 +34,30 @@ Sig is a field for storing public key signatures of the block, typically ed25519

 Note: the max field size is larger than a EdDSA signature (which is what is typically used) in order to allow other primitives for signing in alternative implementations or future versions.

-## signer (optional, necessary if sig is present)
+## signer

 Max byte size: 200

 Signer is a field for specifying the public key which signed the block. In the reference client this is a base64 encoded ed25519 public key.

-## time (mandatory)
+## time

 Max byte size: 10

-Time is an integer field for specifying the time of which a block was created. The trustworthiness of this field is based on one's trust of the block creator, however blocks with a time field set in the future at the point of block receipt (past a reasonable clock skew) are thrown out by the reference client.
+Time is an integer field for specifying the time of which a block was created. The trustworthiness of this field is based on one's trust of the block creator, however blocks with a time field set in the future (past a reasonable clock skew) are thrown out by the reference client.

-## expire (optional)
+## expire

 Max byte size: 10

 Expire is an integer field for specifying the time of which the block creator has indicated that the block should be deleted. The purpose of this is for voluntarily freeing the burden of unwanted blocks on the Onionr network, rather than security/privacy (since blocks could be trivially kept past expiration). Regardless, the reference client deletes blocks after a preset time if the expire field is either not set or longer than the preset time.

-## pow (effectively mandatory)
+## pow

 Max byte size: 1000

 Pow is a field for placing the nonce found to make a block meet a target proof of work. In theory, a block could meet a target without a random token in this field.

-## encryptType (optional)
+## encryptType

-encryptType is a field to specify the mode of encryption for a block. The values supported by Onionr are 'asym' and 'sym'.
+encryptType is a field to specify the mode of encryption for a block. The values supported by Onionr are 'asym' and 'sym'.
--- a/docs/dev/specs/forward-secrecy.md
+++ b/docs/dev/specs/forward-secrecy.md
@ -2,4 +2,4 @@

 # Introduction

-Due to the natural trade-offs of implementing [forward secrecy](https://en.wikipedia.org/wiki/Forward_secrecy) in a distributed, decentralized system, Onionr has optional ephemeral keys.
+Due to the natural trade-offs of implementing [forward secrecy](https://en.wikipedia.org/wiki/Forward_secrecy) in a distributed, decentralized system, Onionr has optional 
--- a/docs/docker.md
+++ b/docs/docker.md
@ -1,28 +0,0 @@
-# Running Onionr in Docker
-
-A Dockerfile is included in the root directory of Onionr.
-
-In Docker version 20.10 (and probably others), there is a strange bug where Onionr must be run with -it or stdout will be garbled and it may hang.
-
-## Clone and build the image
-
-`$ git clone https://git.voidnet.tech/kev/onionr/`
-`$ cd onionr`
-`$ sudo docker build -t onionr .`
-
-
-## Run Onionr
-
-`$ sudo docker run -it -p 8080:8080 onionr`
-
-Onionr will be accessible over any network interface by default, so make sure to either change the entry point bind-address argument or set a firewall rule.
-
-That said, Onionr does protect it's interface by default with a web token, which will be shown in stdout.
-
-**However, anyone who can access the port may be able to see what Onionr sites you have saved and potentially deanonymize your node**
-
-## View the UI
-
-Visit the address and port for the machine Onionr is running on, for example: http://192.168.1.5:8080/#<long-token-taken-from-stdout>
-
-If you want a secure connection to the interface, either use a proxy such as nginx or caddy, or use [SSH tunneling](./vps-cloud-guide.md).
--- a/docs/faq.md
+++ b/docs/faq.md
@ -1,24 +0,0 @@
-<h1 align="center">Onionr FAQ</h1>
-
-(Most of the FAQ answers apply to planned features and do not reflect in the demo network)
-
-* How does Onionr route messages?
-
-Onionr creates a message with an anti-spam proof (usually). It is then forwarded between nodes using binomial tree broadcast.
-
-The network is structured based on the particular transport being used, for Tor, .onion addresses are used to determine closeness between nodes.
-
-Churn due to connectivity changes or issues, membership changes or malicious activity means nodes need to sometimes look far away for missed messages.
-
-To help with scale, messages can be intelligently deleted or forwarded, for example a mail message can be marked by its recipient at a given point for deletion, or forum posts can be set to expire after a short period or after lack of activity.
-
-
-* Why do you use Python instead of [language]?
-
-I'm very comfortable in Python, and I believe it is a maintainable language if you use it correctly with modern features, also it has many quality libraries useful to the project, like pyncal and stem. Most places in the project are IO bound.
-
-
-* What do you think of bad actors who might use Onionr?
-
-Users should be able to exclude viewing, processing, and hosting content they do not want to. To this end, I want to enable user-configurable filtering based on lists provided by friends, fuzzy hashing, and voting.
-
--- a/docs/onionr-1.png
+++ b/docs/onionr-1.png
--- a/docs/onionr-2.png
+++ b/docs/onionr-2.png
--- a/docs/onionr-3.png
+++ b/docs/onionr-3.png
--- a/docs/sticker.png
+++ b/docs/sticker.png
--- a/docs/talk.png
+++ b/docs/talk.png
--- a/docs/tor-stinks-02.png
+++ b/docs/tor-stinks-02.png
--- a/docs/usage/deterministic.md
+++ b/docs/usage/deterministic.md
@ -1,7 +0,0 @@
-# Deterministic address
-
-Deterministic addresses are simply Onionr user IDs that are generated from a passphrase rather than general entropy.
-
-To generate an Onionr deterministic, create a random passphrase of at least 25 characters of length. **It is very important that your passphrase is highly random. 9+ words is recommended for your passphrase**
-
-`$ onionr add-id true`
--- a/docs/usage/firststeps.md
+++ b/docs/usage/firststeps.md
@ -2,6 +2,6 @@

 After installing Onionr, there are several things to do:

-1. Setup a [deterministic address](deterministic.md) (optional)
-2. Start your node `/path/to/onionr.sh start`
-3. Copy your ID and share it with friends
+1. Setup a [deterministic address](usage/deterministic.md) (optional)
+2. Add friends' ids
+3. Publish your id
--- a/docs/usage/install.md
+++ b/docs/usage/install.md
@ -1,18 +1,13 @@
 # Onionr Installation

-The following steps work broadly speaking for WSL, Mac, and Linux.
+The following steps work broadly speaking for Windows, Mac, and Linux.

-1. Verify python3.7+ is installed: if not, see https://www.python.org/downloads/
+1. Verify python3.6+ is installed: if its not see https://www.python.org/downloads/

 2. Verify Tor is installed (does not need to be running, binary can be put into system path or Onionr directory)

 3. [Optional but recommended]: setup virtual environment using [virtualenv](https://virtualenv.pypa.io/en/latest/), activate the virtual environment

-4. Clone Onionr: `$ git clone https://gitlab.com/beardog/onionr`
+4. Clone Onionr: git clone https://gitlab.com/beardog/onionr

-5. Install the Python module dependencies: `$ pip3 install --require-hashes -r requirements.txt`
-
-
-Note: if an alternative python install is needed, use virtualenv or run Onionr commands with:
-
-`$ /path/to/python /path/to/onionr/onionr/__init__.py`
+5. Install the Python module dependencies: pip3 install --require-hashes -r requirements.txt
--- a/docs/usage/mail.md
+++ b/docs/usage/mail.md
@ -1,16 +0,0 @@
-To use OnionrMail, open the web interface:
-
-`$ onionr.sh openweb`
-
-Navigate to the mail interface from the navbar.
-
-## Receiving mail
-
-Copy your ID from the textbox below the navbar. Publish the ID so that someone (or general people) know your ID to send messages to it. Note that associating the ID with your real ID publicly removes anonymity.
-
-## Sending mail
-
-1. optionally add the recipient as a friend via the friends page.
-2. click compose in the mail menu
-3. either select the recipient from the friend list or paste their ID in manually into the recipient box
-4. send away!
--- a/docs/usage/pages.md
+++ b/docs/usage/pages.md
@ -1,59 +0,0 @@
-Onionr sites come in two forms:
-
-* Single-page sites, identified by the hash of a single page contained within a single Onionr block.
-
-* Multi-page sites, identified by a user ID. Contains directory archives of a full site.
-
-
-# Metadata Awareness
-
-Before creating an Onionr site, one should be cautious of the metadata one could be leaking. For example, some HTML generators may insert author meta tags. Onionr does not filter out any web page data.
-
-# No JavaScript, no third-party resources
-
-Currently, in order to protect Onionr users, JavaScript is disabled within Onionr sites. JS will remain present in the HTML file, but be non functional. Additionally, third party resources outside of Onionr cannot be loaded.
-
-
-# Creating multi page sites
-
-Multi page sites are the most useful, as they can contain an arbitrary amount of static files.
-
-To create a single page site, create a directory for your site and write standard HTML file(s) within them. CSS, images and other files can be placed in the directory as well. The home page should be name index.html and in the parent level directory.
-
-Then, create a strong passphrase for the site. If the site will be updated, be sure to write it down or remember it. A strong passphrase can be generated by running:
-
-`$ scripts/passphrase-generator.py`
-
-Sample output: lovesick blubberer haemoglobin... and so on.
-
-## Generating or updating the site:
-
-`$ ./onionr.sh addsite`
-
-All files in the current working directory will be added to the site.
-
-The command will prompt for a passphrase.
-
-After the site is generated, a user ID that identifies the site will be outputted.
-
-# Creating single page sites
-
-Single page sites are incredibly straight forward.
-
-Single page sites cannot be modified or updated, but are somewhat more secure due to having lower complexity.
-
-To create a single page site, write a standard HTML file. Inline or data-uri CSS can be included, as well as data-uri images. Data-URI generators can be found online.
-
-After creating the HTML file, run this command:
-
-`$ ./onionr.sh addhtml filename.html`
-
-![single page screenshot](single-page.png)
-
-# Viewing sites
-
-To view a site, open the Onionr web interface and paste the site hash or ID into the site opener box that looks like this:
-
-![site opener box screenshot](site-opener.png)
-
-Then, press open.
--- a/docs/usage/safety.md
+++ b/docs/usage/safety.md
@ -1,6 +0,0 @@
-Onionr and all privacy-oriented software are not silver bullets.
-
-For example, if one gives away too much about themselves on an Onionr forum, it doesn't matter how good Onionr's privacy is.
-
-
-TODO: expand this page
--- a/docs/usage/single-page.png
+++ b/docs/usage/single-page.png
--- a/docs/usage/site-opener.png
+++ b/docs/usage/site-opener.png
--- a/docs/vps-cloud-guide.md
+++ b/docs/vps-cloud-guide.md
@ -1,18 +0,0 @@
-# Cloud/Server Hosting Onionr
-
-Cloud hosting is not the recommended way to run Onionr, as it gives the cloud provider control over your data.
-
-That said, it is quite useful and by running a 24/7 Onionr node you contribute to the network's health.
-
-![https://cock.li/img/cockbox.png](https://cockbox.org/?r=12)
-
-[Cockbox](https://cockbox.org/?r=12) is the recommended provider, as they do not demand personal information and accept Monero. This is an affiliate link, which gives us a commission at no expense or complication to you.
-
-
-1. [Install Onionr like normal](./basic-onionr-user-guide.pdf) or [from Docker](./docker.md)
-2. Run with specific host and port to bind to: `$ ./run-onionr-node.py --bind-address 0.0.0.0 --port 8080`
-3. Get the web security token: `$ ./onionr.sh get-web` (the long string at the end of the URL)
-4. [Configure Firefox for SSH tunneling](https://web.archive.org/web/20210123034529/https://gist.github.com/brentjanderson/6ed800376e53746d2d28ba7b6bdcdc12). `Set network.proxy.allow_hijacking_localhost` to true in about:config
-
-
-Disable random host binding in config (general.security.random_bind_ip = false) if you have Onionr auto start. You may also want to disable deniable block inserts.
--- a/docs/whitepaper.md
+++ b/docs/whitepaper.md
@ -1,18 +1,15 @@
 <p align="center">
-   <h1 align="center">Onionr</h1>
+ <img src="onionr-logo.png" alt="<h1>Onionr</h1>" width=200>
 </p>
-
 <p align="center">Anonymous, Decentralized, Distributed Network</p>

-June 2020
-
 # Introduction

 We believe that the ability to communicate freely with others is crucial for maintaining societal and personal liberty. The internet has provided humanity with the ability to spread information globally, but there are many persons and organizations who try to stifle the flow of information, sometimes with success.

-Internet censorship comes in many forms, state censorship, threats of violence, network exploitation (e.g. denial of service attacks) and others.
+Internet censorship comes in many forms, state censorship, corporate consolidation of media, threats of violence, network exploitation (e.g. denial of service attacks) and other threats.

-We hold that in order to protect individual privacy, users must have the ability to communicate anonymously and with decentralization.
+We hold that in order to protect individual privacy, users must have the ability to communicate anonymously and with decentralization. 

 We believe that in order to prevent censorship and loss of information, these measures must be in place:

@ -60,14 +57,12 @@ To mitigate maliciously slow or unreliable nodes, Onionr builds a profile on nod

 ## Block Format

-Onionr blocks are very simple. They are structured in two main parts: a metadata section and a data section, with a line feed delimiting where metadata ends and data begins.
+Onionr blocks are very simple. They are structured in two main parts: a metadata section and a data section, with a line feed delimiting where metadata ends and data begins. 

 Metadata defines what kind of data is in a block, signature data, encryption settings, and other arbitrary information.

 Optionally, a random token can be inserted into the metadata for use in Proof of Work.

-The proof of work function should be a Verifiable Delay Function (VDF). We have chosen MiMC for it's simplicity.
-
 ### Block Encryption

 For encryption, Onionr uses ephemeral Curve25519 keys for key exchange and XSalsa20-Poly1305 as a symmetric cipher or optionally using only XSalsa20-Poly1305 with a pre-shared key.
@ -88,7 +83,9 @@ Blocks are stored indefinitely until the allocated space is filled, at which poi

 ## Block Timestamping

-Onionr blocks are by default not accepted if their timestamp is set too far in the past, or is in the future.
+Onionr can provide evidence of when a block was inserted by requesting other users to sign a hash of the current time with the block data hash: sha3_256(time + sha3_256(block data)).
+
+This can be done either by the creator of the block prior to generation, or by any node after insertion.

 In addition, randomness beacons such as the one operated by [NIST](https://beacon.nist.gov/home), [Chile](https://beacon.clcert.cl/), or the hash of the latest blocks in a cryptocurrency network could be used to affirm that a block was at least not *created* before a given time.

@ -100,7 +97,7 @@ The benefits of such a system are increased privacy, and the ability to anonymou

 # Threat Model

-The goal of Onionr is to provide a method of distributing information in a manner in which the difficulty of discovering the identity of those sending and receiving the information is greatly increased. In this section we detail what information we want to protect and who we're protecting it from.
+The goal of Onionr is to provide a method of distributing information in a manner in which the difficulty of discovering the identity of those sending and receiving the information is greatly increased. In this section we detail what information we want to protect and who we're protecting it from. 

 In this threat model, "protected" means available in plaintext only to those which it was intended, and regardless non-malleable

@ -134,7 +131,7 @@ Onionr does not protect the following:

 ## Assumptions

-We assume that Tor onion services (v3) and I2P services cannot be trivially deanonymized, and that the underlying cryptographic primitives we employ cannot be broken in any manner faster than brute force unless a quantum computer is used.
+We assume that Tor onion services (v3) and I2P services cannot be trivially deanonymized, and that the underlying cryptographic primitives we employ cannot be broken in any manner faster than brute force unless a quantum computer is used. 

 Once quantum safe algorithms are more mature and have decent high level libraries, they will be deployed.

--- a/git-update.sh
+++ b/git-update.sh
@ -1,9 +0,0 @@
-#!/bin/bash
-
-echo "This will git pull origin master, then update dependencies and default plugins."
-echo "Enter to continue, ctrl-c to stop."
-read
-
-git pull origin master
-pip3 install -r --require-hashes requirements.txt
-./onionr.sh reset-plugins
--- a/install/CentOS-install
+++ b/install/CentOS-install
@ -1,35 +0,0 @@
-yum install epel-release
-yum clean all
-yum update -y
-
-yum group install "Development Tools"
-yum install centos-release-scl
-yum install wget zlib zlib-devel openssl-devel bzip2-devel libffi-devel python-gevent -y
-yum install rh-python38 rh-python38-python-devel
-scl enable rh-python38 bash
-yum update -y
-
-pip3 install --require-hashes -r requirements.txt
-chown nobody. ../
-su -l nobody -s /bin/bash
-scl enable rh-python38 bash
-./onionr.sh start
-
-Like a black hole, NSA pulls in every signal that comes near, but no electron is ever allowed to escape│
-│ -James Bamford                                                                                        │
-└───────────────────────────────────────────────────────────────────────────────────────────────────────┘
-[+] Tor is starting...
-[+] Jan 11 18:35:55.000 [notice] Bootstrapped 0%: Starting
-[+] Jan 11 18:35:56.000 [notice] Bootstrapped 10%: Finishing handshake with directory server
-[+] Jan 11 18:35:56.000 [notice] Bootstrapped 80%: Connecting to the Tor network
-[+] Jan 11 18:35:56.000 [notice] Bootstrapped 90%: Establishing a Tor circuit
-[+] Finished starting Tor.
-[+] Onionr v8.0.1 (x86_64) (API v2)
-[+] Private P2P Communication - GPLv3 - https://Onionr.net
-[+] CPython 3.8.6 on Linux-3.10.0-1062.4.1.el7.x86_64-x86_64-with-glibc2.2.5 3.10.0-1062.4.1.el7.x86_64
-[+] Onionr data dir: /root/.local/share/onionr/
-[+] Onionr daemon is running under 5186
-[+] Not binding to LAN due to no private network configured.
-[+] First run detected. Run openhome to get setup.
-
-./onionr.sh openhome
--- a/install/README.md
+++ b/install/README.md
@ -1,5 +0,0 @@
-# Service/System Installation Notes
-
-Currently, the most reliable and well tested way to use Onionr is straight from the master branch of the repository, or from the released bundles.
-
-System services and system-wide installation is not supported well yet.
--- a/install/install_arch.sh
+++ b/install/install_arch.sh
@ -80,7 +80,7 @@ systemctl start onionr

 # pretty header thing

-"$EXECUTABLE" details
+"$EXECUTABLE" --header 'Onionr successfully installed.'

 # and we're good!

--- a/install/onionr
+++ b/install/onionr
@ -1,16 +1,23 @@
 #!/bin/sh
+
 set -e

-if [[ $EUID -eq 0 ]]; then
-    export ONIONR_HOME=${ONIONR_HOME:-/var/lib/onionr}
-    export LOG_DIR=${LOG_DIR:-/var/log/onionr}
+export OUTPUT_DIR=${OUTPUT_DIR:=/usr/share/onionr}
+
+if [ -n "$HOME" ]; then
+    export XDG_DATA_HOME=${XDG_DATA_HOME:=$HOME/.local/share/onionr}
+
+    export ONIONR_HOME=${ONIONR_HOME:=$XDG_DATA_HOME}
+    export LOG_DIR=${LOG_DIR:=$XDG_DATA_HOME/logs}
 else
-    export ONIONR_HOME=${ONIONR_HOME:-${XDG_DATA_HOME:-$HOME/.local/share}/onionr}
-    export LOG_DIR=${LOG_DIR:-$ONIONR_HOME/logs}
+    export ONIONR_HOME=${ONIONR_HOME:=/etc/onionr}
+    export LOG_DIR=${LOG_DIR:=/var/log/onionr}
 fi

 mkdir -p "$ONIONR_HOME" "$LOG_DIR"

-chmod 0700 "$ONIONR_HOME" "$LOG_DIR"
+chmod -R 700 "$ONIONR_HOME" "$LOG_DIR"
+chown -R $USER:$USER "$ONIONR_HOME" "$LOG_DIR"

-exec ${ONIONR_BASEDIR:-/usr/share/onionr}/onionr.sh "$@"
+cd "$OUTPUT_DIR/onionr"
+exec python3.7 onionr.py "$@"
--- a/install/onionr.service
+++ b/install/onionr.service
@ -1,20 +1,21 @@
 [Unit]
 Description=Onionr Daemon
 Documentation=https://onionr.net/docs/
-After=network-online.target
-Requires=network-online.target
+After=network.target tor.service
+Requires=network.target tor.service systemd-networkd-wait-online.service

 [Service]
-Environment="ONIONR_HOME=/var/lib/onionr"
-Environment="LOG_DIR=/var/log/onionr"
+Environment="DATA_DIR=/usr/share/onionr"
+Environment="LOG_DIR=/var/log/onionr/"

-ExecStart=/usr/bin/onionr start
-ExecStop=/usr/bin/onionr stop
+ExecStart=/usr/bin/onionr --start
+ExecStop=/usr/bin/onionr --stop

 KillMode=mixed
 KillSignal=SIGQUIT
-TimeoutStopSec=30s
+TimeoutStopSec=5s
+Type=simple
 Restart=on-abnormal

 [Install]
-WantedBy=multi-user.target
+WantedBy=tor.service
--- a/make-release.sh
+++ b/make-release.sh
@ -1,15 +0,0 @@
-#!/usr/bin/bash
-rm -rf dist
-mkdir dist
-mkdir dist/onionr/
-cp -t dist/onionr/ -r docs static-data install src onionr.sh start-daemon.sh setprofile.sh requirements.txt requirements-notifications.txt
-cp *.md dist/onionr/
-PIP_USER=false
-export PIP_USER
-cd dist
-tar -czvf onionr-no-deps.tar.gz onionr
-cd ..
-pip3 install --require-hashes -r requirements.txt --target=dist/onionr/src/
-pip3 install --require-hashes -r requirements-notifications.txt --target=dist/onionr/src/
-cd dist
-tar -czvf onionr.tar.gz onionr
--- a/onionr.sh
+++ b/onionr.sh
@ -1,9 +1,6 @@
 #!/bin/sh
 ORIG_ONIONR_RUN_DIR=`pwd`
 export ORIG_ONIONR_RUN_DIR
-export PYTHONDONTWRITEBYTECODE=1
-export PYTHONUNBUFFERED=1
-export PYTHONOPTIMIZE="true"
 cd "$(dirname "$0")"
-cd src
-./__init__.py "$@"
+cd onionr/
+./__init__.py "$@"
--- a/onionr/init.py
+++ b/onionr/init.py
@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+'''
+    Onionr - Private P2P Communication
+
+    This file initializes Onionr when ran to be a daemon or with commands
+
+    Run with 'help' for usage.
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+# Set the user's locale for encoding reasons
+import locale 
+locale.setlocale(locale.LC_ALL, '')
+
+# Import standard libraries
+import sys
+
+# 3rd party lib imports
+# Ensure that PySocks is installed
+try:
+    from urllib3.contrib.socks import SOCKSProxyManager
+except ModuleNotFoundError:
+    raise ModuleNotFoundError("You need the PySocks module (for use with socks5 proxy to use Tor)")
+
+# Onionr imports
+from etc import onionrvalues # For different Onionr related constants such as versions
+import onionrsetup as setup
+
+# Ensure we have at least the minimum python version
+if sys.version_info[0] == 2 or sys.version_info[1] < onionrvalues.MIN_PY_VERSION:
+    sys.stderr.write('Error, Onionr requires Python 3.%s+\n' % (onionrvalues.MIN_PY_VERSION,))
+    sys.exit(1)
+
+# Create Onionr data directories, must be done before most imports
+from utils import createdirs
+createdirs.create_dirs()
+
+from onionrcommands import parser
+import onionrevents as events
+
+setup.setup_config()
+setup.setup_default_plugins()
+
+def onionr_main():
+    parser.register()
+    return
+
+if __name__ == "__main__":
+    onionr_main()
--- a/onionr/apiservers/README.md
+++ b/onionr/apiservers/README.md
@ -0,0 +1,9 @@
+# API Servers
+
+Contains the WSGI servers Onionr uses for remote peer communication and local daemon control
+
+## Files
+
+* \_\_init\_\_.py: Exposes the server classes
+* private: Contains the client API (the server used to interact with the local Onionr daemon, and view the web UI)
+* public: Contains the public API (the server used by remote peers to talk to our daemon)
--- a/onionr/apiservers/init.py
+++ b/onionr/apiservers/init.py
@ -0,0 +1,3 @@
+from . import public, private
+PublicAPI = public.PublicAPI
+ClientAPI = private.PrivateAPI
--- a/onionr/apiservers/private/README.md
+++ b/onionr/apiservers/private/README.md
@ -0,0 +1,8 @@
+# Private API Server
+
+Private API server, used to access the web interface locally and control Onionr
+
+## Files
+
+* \_\_init\_\_.py: Sets up the server and a few misc functions
+* register_private_blueprints.py: Adds in flask blueprints for various sub-APIs
--- a/onionr/apiservers/private/init.py
+++ b/onionr/apiservers/private/init.py
@ -0,0 +1,95 @@
+'''
+    Onionr - Private P2P Communication
+
+    This file handles all incoming http requests to the client, using Flask
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import base64, os, time
+import flask
+from gevent.pywsgi import WSGIServer
+from onionrutils import epoch
+import httpapi, filepaths, logger
+from . import register_private_blueprints
+from etc import waitforsetvar
+import serializeddata, config
+from .. import public
+class PrivateAPI:
+    '''
+        Client HTTP api
+    '''
+
+    callbacks = {'public' : {}, 'private' : {}}
+
+    def __init__(self):
+        '''
+            Initialize the api server, preping variables for later use
+
+            This initialization defines all of the API entry points and handlers for the endpoints and errors
+            This also saves the used host (random localhost IP address) to the data folder in host.txt
+        '''
+        self.config = config
+        self.startTime = epoch.get_epoch()
+        app = flask.Flask(__name__)
+        bindPort = int(config.get('client.client.port', 59496))
+        self.bindPort = bindPort
+
+        self.clientToken = config.get('client.webpassword')
+        self.timeBypassToken = base64.b16encode(os.urandom(32)).decode()
+
+        self.host = httpapi.apiutils.setbindip.set_bind_IP(filepaths.private_API_host_file)
+        logger.info('Running api on %s:%s' % (self.host, self.bindPort))
+        self.httpServer = ''
+
+        self.queueResponse = {}
+        self.get_block_data = httpapi.apiutils.GetBlockData(self)
+        register_private_blueprints.register_private_blueprints(self, app)
+        httpapi.load_plugin_blueprints(app)
+        self.app = app
+    
+    def start(self):
+        waitforsetvar.wait_for_set_var(self, "_too_many")
+        self.publicAPI = self._too_many.get(public.PublicAPI)
+        self.httpServer = WSGIServer((self.host, self.bindPort), self.app, log=None, handler_class=httpapi.fdsafehandler.FDSafeHandler)
+        self.httpServer.serve_forever()
+
+    def setPublicAPIInstance(self, inst):
+        self.publicAPI = inst
+
+    def validateToken(self, token):
+        '''
+            Validate that the client token matches the given token. Used to prevent CSRF and data exfiltration
+        '''
+        if not self.clientToken:
+            logger.error("client password needs to be set")
+            return False
+        try:
+            if not hmac.compare_digest(self.clientToken, token):
+                return False
+            else:
+                return True
+        except TypeError:
+            return False
+
+    def getUptime(self):
+        while True:
+            try:
+                return epoch.get_epoch() - self.startTime
+            except (AttributeError, NameError):
+                # Don't error on race condition with startup
+                pass
+
+    def getBlockData(self, bHash, decrypt=False, raw=False, headerOnly=False):
+        return self.get_block_data.get_block_data(bHash, decrypt=decrypt, raw=raw, headerOnly=headerOnly)
--- a/onionr/apiservers/private/register_private_blueprints.py
+++ b/onionr/apiservers/private/register_private_blueprints.py
@ -0,0 +1,35 @@
+'''
+    Onionr - Private P2P Communication
+
+    This file registers blueprints for the private api server
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import os
+from httpapi import security, friendsapi, profilesapi, configapi, insertblock, miscclientapi, onionrsitesapi, apiutils
+from httpapi import directconnections
+def register_private_blueprints(private_api, app):
+    app.register_blueprint(security.client.ClientAPISecurity(private_api).client_api_security_bp)
+    app.register_blueprint(friendsapi.friends)
+    app.register_blueprint(profilesapi.profile_BP)
+    app.register_blueprint(configapi.config_BP)
+    app.register_blueprint(insertblock.ib)
+    app.register_blueprint(miscclientapi.getblocks.client_get_blocks)
+    app.register_blueprint(miscclientapi.endpoints.PrivateEndpoints(private_api).private_endpoints_bp)
+    app.register_blueprint(onionrsitesapi.site_api)
+    app.register_blueprint(apiutils.shutdown.shutdown_bp)
+    app.register_blueprint(miscclientapi.staticfiles.static_files_bp)
+    app.register_blueprint(directconnections.DirectConnectionManagement(private_api).direct_conn_management_bp)
+    return app
--- a/onionr/apiservers/public/init.py
+++ b/onionr/apiservers/public/init.py
@ -0,0 +1,64 @@
+'''
+    Onionr - Private P2P Communication
+
+    This file handles all incoming http requests to the public api server, using Flask
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import time
+import threading
+import flask
+from gevent.pywsgi import WSGIServer
+from httpapi import apiutils, security, fdsafehandler, miscpublicapi
+import logger, config, filepaths
+from utils import gettransports
+from etc import onionrvalues, waitforsetvar
+
+def _get_tor_adder(pub_api):
+    transports = []
+    while len(transports) == 0:
+        transports = gettransports.get()
+        time.sleep(0.3)
+    pub_api.torAdder = transports[0]
+
+class PublicAPI:
+    '''
+        The new client api server, isolated from the public api
+    '''
+    def __init__(self):
+        app = flask.Flask('PublicAPI')
+        app.config['MAX_CONTENT_LENGTH'] = 5 * 1024 * 1024
+        self.i2pEnabled = config.get('i2p.host', False)
+        self.hideBlocks = [] # Blocks to be denied sharing
+        self.host = apiutils.setbindip.set_bind_IP(filepaths.public_API_host_file)
+
+        threading.Thread(target=_get_tor_adder, args=[self], daemon=True).start()
+
+        self.torAdder = ""
+        self.bindPort = config.get('client.public.port')
+        self.lastRequest = 0
+        self.hitCount = 0 # total rec requests to public api since server started
+        self.config = config
+        self.API_VERSION = onionrvalues.API_VERSION
+        logger.info('Running public api on %s:%s' % (self.host, self.bindPort))
+
+        app.register_blueprint(security.public.PublicAPISecurity(self).public_api_security_bp)
+        app.register_blueprint(miscpublicapi.endpoints.PublicEndpoints(self).public_endpoints_bp)
+        self.app = app
+
+    def start(self):
+        waitforsetvar.wait_for_set_var(self, "_too_many")
+        self.httpServer = WSGIServer((self.host, self.bindPort), self.app, log=None, handler_class=fdsafehandler.FDSafeHandler)
+        self.httpServer.serve_forever()
--- a/onionr/blockimporter.py
+++ b/onionr/blockimporter.py
@ -0,0 +1,54 @@
+'''
+    Onionr - Private P2P Communication
+
+    Import block data and save it
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import onionrexceptions, logger
+from onionrutils import validatemetadata, blockmetadata
+from coredb import blockmetadb
+import onionrblacklist, onionrstorage
+import onionrcrypto as crypto
+def importBlockFromData(content):
+    blacklist = onionrblacklist.OnionrBlackList()
+    retData = False
+
+    try:
+        content = content.encode()
+    except AttributeError:
+        pass
+
+    dataHash = crypto.hashers.sha3_hash(content)
+
+    if blacklist.inBlacklist(dataHash):
+        raise onionrexceptions.BlacklistedBlock('%s is a blacklisted block' % (dataHash,))
+
+    metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
+    metadata = metas[0]
+    if validatemetadata.validate_metadata(metadata, metas[2]): # check if metadata is valid
+        if crypto.cryptoutils.verify_POW(content): # check if POW is enough/correct
+            logger.info('Block passed proof, saving.', terminal=True)
+            try:
+                blockHash = onionrstorage.set_data(content)
+            except onionrexceptions.DiskAllocationReached:
+                logger.warn('Failed to save block due to full disk allocation')
+            else:
+                blockmetadb.add_to_block_DB(blockHash, dataSaved=True)
+                blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database
+                retData = True
+        else:
+            raise onionrexceptions.InvalidProof
+    return retData
--- a/onionr/communicator/README.md
+++ b/onionr/communicator/README.md
@ -0,0 +1,15 @@
+# Onionr Communicator
+
+Onionr communicator is the Onionr client. It "connects" to remote Onionr peers and does things such as:
+
+* Finding new peers
+* Uploading blocks
+* Downloading blocks
+* Daemon maintenance/housekeeping
+
+## Files
+
+* \_\_init\_\_.py: Contains the main communicator code. Inits and launches the communicator and sets up the timers
+* peeraction.py: contains a function to send commands to remote peers
+* bootstrappers.py: adds peers from the bootstrap list to the communicator to try to connect to them
+* onlinepeers: management of the online peer pool for the communicator
--- a/onionr/communicator/init.py
+++ b/onionr/communicator/init.py
@ -0,0 +1,261 @@
+'''
+    Onionr - Private P2P Communication
+
+    This file contains both the OnionrCommunicate class for communcating with peers
+    and code to operate as a daemon, getting commands from the command queue database (see core.Core.daemonQueue)
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sys, os, time
+import config, logger
+import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block
+from . import onlinepeers, uploadqueue
+from communicatorutils import servicecreator, onionrcommunicatortimers
+from communicatorutils import downloadblocks, lookupblocks, lookupadders
+from communicatorutils import servicecreator, connectnewpeers
+from communicatorutils import uploadblocks
+from communicatorutils import daemonqueuehandler, announcenode, deniableinserts
+from communicatorutils import cooldownpeer, housekeeping, netcheck
+from onionrutils import localcommand, epoch
+from etc import humanreadabletime
+import onionrservices, filepaths, storagecounter
+from coredb import daemonqueue, dbfiles
+from utils import gettransports
+from netcontroller import NetController
+OnionrCommunicatorTimers = onionrcommunicatortimers.OnionrCommunicatorTimers
+
+config.reload()
+class OnionrCommunicatorDaemon:
+    def __init__(self, shared_state, developmentMode=config.get('general.dev_mode', False)):
+        # configure logger and stuff
+        self.config = config
+        self.storage_counter = storagecounter.StorageCounter()
+        self.isOnline = True # Assume we're connected to the internet
+        self.shared_state = shared_state # TooManyObjects module
+
+        # list of timer instances
+        self.timers = []
+
+        # initialize core with Tor socks port being 3rd argument
+        self.proxyPort = shared_state.get(NetController).socksPort
+
+        # Upload information, list of blocks to upload
+        self.blocksToUpload = []
+        self.upload_session_manager = self.shared_state.get(uploadblocks.sessionmanager.BlockUploadSessionManager)
+        self.shared_state.share_object()
+
+        # loop time.sleep delay in seconds
+        self.delay = 1
+
+        # lists of connected peers and peers we know we can't reach currently
+        self.onlinePeers = []
+        self.offlinePeers = []
+        self.cooldownPeer = {}
+        self.connectTimes = {}
+        self.peerProfiles = [] # list of peer's profiles (onionrpeers.PeerProfile instances)
+        self.newPeers = [] # Peers merged to us. Don't add to db until we know they're reachable
+        self.announceProgress = {}
+        self.announceCache = {}
+
+        self.generating_blocks = []
+
+        # amount of threads running by name, used to prevent too many
+        self.threadCounts = {}
+
+        # set true when shutdown command received
+        self.shutdown = False
+
+        # list of new blocks to download, added to when new block lists are fetched from peers
+        self.blockQueue = {}
+
+        # list of blocks currently downloading, avoid s
+        self.currentDownloading = []
+
+        # timestamp when the last online node was seen
+        self.lastNodeSeen = None
+
+        # Dict of time stamps for peer's block list lookup times, to avoid downloading full lists all the time
+        self.dbTimestamps = {}
+
+        # Clear the daemon queue for any dead messages
+        if os.path.exists(dbfiles.daemon_queue_db):
+            daemonqueue.clear_daemon_queue()
+
+        # Loads in and starts the enabled plugins
+        plugins.reload()
+
+        # time app started running for info/statistics purposes
+        self.startTime = epoch.get_epoch()
+
+        uploadqueue.UploadQueue(self) # extends our upload list and saves our list when Onionr exits
+
+        if developmentMode:
+            OnionrCommunicatorTimers(self, self.heartbeat, 30)
+
+        # Set timers, function reference, seconds
+        # requires_peer True means the timer function won't fire if we have no connected peers
+        peerPoolTimer = OnionrCommunicatorTimers(self, onlinepeers.get_online_peers, 60, max_threads=1, my_args=[self])
+        OnionrCommunicatorTimers(self, self.runCheck, 2, max_threads=1)
+
+        # Timers to periodically lookup new blocks and download them
+        lookup_blocks_timer = OnionrCommunicatorTimers(self, lookupblocks.lookup_blocks_from_communicator, config.get('timers.lookupBlocks', 25), my_args=[self], requires_peer=True, max_threads=1)
+        # The block download timer is accessed by the block lookup function to trigger faster download starts
+        self.download_blocks_timer = OnionrCommunicatorTimers(self, self.getBlocks, config.get('timers.getBlocks', 10), requires_peer=True, max_threads=5)
+
+        # Timer to reset the longest offline peer so contact can be attempted again
+        OnionrCommunicatorTimers(self, onlinepeers.clear_offline_peer, 58, my_args=[self])
+
+        # Timer to cleanup old blocks
+        blockCleanupTimer = OnionrCommunicatorTimers(self, housekeeping.clean_old_blocks, 20, my_args=[self])
+
+        # Timer to discover new peers
+        OnionrCommunicatorTimers(self, lookupadders.lookup_new_peer_transports_with_communicator, 60, requires_peer=True, my_args=[self], max_threads=2)
+
+        # Timer for adjusting which peers we actively communicate to at any given time, to avoid over-using peers
+        OnionrCommunicatorTimers(self, cooldownpeer.cooldown_peer, 30, my_args=[self], requires_peer=True)
+
+        # Timer to read the upload queue and upload the entries to peers
+        OnionrCommunicatorTimers(self, uploadblocks.upload_blocks_from_communicator, 5, my_args=[self], requires_peer=True, max_threads=1)
+
+        # Timer to process the daemon command queue
+        OnionrCommunicatorTimers(self, daemonqueuehandler.handle_daemon_commands, 6, my_args=[self], max_threads=3)
+
+        # Setup direct connections
+        if config.get('general.socket_servers', False):
+            self.services = onionrservices.OnionrServices()
+            self.active_services = []
+            self.service_greenlets = []
+            OnionrCommunicatorTimers(self, servicecreator.service_creator, 5, max_threads=50, my_args=[self])
+        else:
+            self.services = None
+        
+        # {peer_pubkey: ephemeral_address}, the address to reach them
+        self.direct_connection_clients = {}
+        
+        # This timer creates deniable blocks, in an attempt to further obfuscate block insertion metadata
+        if config.get('general.insert_deniable_blocks', True):
+            deniableBlockTimer = OnionrCommunicatorTimers(self, deniableinserts.insert_deniable_block, 180, my_args=[self], requires_peer=True, max_threads=1)
+            deniableBlockTimer.count = (deniableBlockTimer.frequency - 175)
+
+        # Timer to check for connectivity, through Tor to various high-profile onion services
+        netCheckTimer = OnionrCommunicatorTimers(self, netcheck.net_check, 500, my_args=[self], max_threads=1)
+
+        # Announce the public API server transport address to other nodes if security level allows
+        if config.get('general.security_level', 1) == 0 and config.get('general.announce_node', True):
+            # Default to high security level incase config breaks
+            announceTimer = OnionrCommunicatorTimers(self, announcenode.announce_node, 3600, my_args=[self], requires_peer=True, max_threads=1)
+            announceTimer.count = (announceTimer.frequency - 120)
+        else:
+            logger.debug('Will not announce node.')
+        
+        # Timer to delete malfunctioning or long-dead peers
+        cleanupTimer = OnionrCommunicatorTimers(self, self.peerCleanup, 300, requires_peer=True)
+
+        # Timer to cleanup dead ephemeral forward secrecy keys 
+        forwardSecrecyTimer = OnionrCommunicatorTimers(self, housekeeping.clean_keys, 15, my_args=[self], max_threads=1)
+
+        # Adjust initial timer triggers
+        peerPoolTimer.count = (peerPoolTimer.frequency - 1)
+        cleanupTimer.count = (cleanupTimer.frequency - 60)
+        blockCleanupTimer.count = (blockCleanupTimer.frequency - 2)
+        lookup_blocks_timer = (lookup_blocks_timer.frequency - 2)
+
+        shared_state.add(self)
+
+        # Main daemon loop, mainly for calling timers, don't do any complex operations here to avoid locking
+        try:
+            while not self.shutdown:
+                for i in self.timers:
+                    if self.shutdown:
+                        break
+                    i.processTimer()
+                time.sleep(self.delay)
+                # Debug to print out used FDs (regular and net)
+                #proc = psutil.Process()
+                #print(proc.open_files(), len(psutil.net_connections()))
+        except KeyboardInterrupt:
+            self.shutdown = True
+            pass
+
+        logger.info('Goodbye. (Onionr is cleaning up, and will exit)', terminal=True)
+        try:
+            self.service_greenlets
+        except AttributeError:
+            pass
+        else:
+            for server in self.service_greenlets:
+                server.stop()
+        localcommand.local_command('shutdown') # shutdown the api
+        try:
+            time.sleep(0.5)
+        except KeyboardInterrupt:
+            pass
+
+    def getBlocks(self):
+        '''download new blocks in queue'''
+        downloadblocks.download_blocks_from_communicator(self)
+
+    def decrementThreadCount(self, threadName):
+        '''Decrement amount of a thread name if more than zero, called when a function meant to be run in a thread ends'''
+        try:
+            if self.threadCounts[threadName] > 0:
+                self.threadCounts[threadName] -= 1
+        except KeyError:
+            pass
+
+    def connectNewPeer(self, peer='', useBootstrap=False):
+        '''Adds a new random online peer to self.onlinePeers'''
+        connectnewpeers.connect_new_peer_to_communicator(self, peer, useBootstrap)
+
+    def peerCleanup(self):
+        '''This just calls onionrpeers.cleanupPeers, which removes dead or bad peers (offline too long, too slow)'''
+        onionrpeers.peer_cleanup()
+        self.decrementThreadCount('peerCleanup')
+
+    def getPeerProfileInstance(self, peer):
+        '''Gets a peer profile instance from the list of profiles, by address name'''
+        for i in self.peerProfiles:
+            # if the peer's profile is already loaded, return that
+            if i.address == peer:
+                retData = i
+                break
+        else:
+            # if the peer's profile is not loaded, return a new one. connectNewPeer also adds it to the list on connect
+            retData = onionrpeers.PeerProfiles(peer)
+            self.peerProfiles.append(retData)
+        return retData
+
+    def getUptime(self):
+        return epoch.get_epoch() - self.startTime
+
+    def heartbeat(self):
+        '''Show a heartbeat debug message'''
+        logger.debug('Heartbeat. Node running for %s.' % humanreadabletime.human_readable_time(self.getUptime()))
+        self.decrementThreadCount('heartbeat')
+
+    def runCheck(self):
+        if run_file_exists(self):
+            logger.debug('Status check; looks good.')
+
+        self.decrementThreadCount('runCheck')
+
+def startCommunicator(shared_state):
+    OnionrCommunicatorDaemon(shared_state)
+
+def run_file_exists(daemon):
+    if os.path.isfile(filepaths.run_check_file):
+        os.remove(filepaths.run_check_file)
+        return True
+    return False
--- a/onionr/communicator/bootstrappeers.py
+++ b/onionr/communicator/bootstrappeers.py
@ -0,0 +1,30 @@
+'''
+    Onionr - Private P2P Communication
+
+    add bootstrap peers to the communicator peer list
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+from utils import readstatic, gettransports
+from coredb import keydb
+bootstrap_peers = readstatic.read_static('bootstrap-nodes.txt').split(',')
+def add_bootstrap_list_to_peer_list(comm_inst, peerList):
+    '''
+        Add the bootstrap list to the peer list (no duplicates)
+    '''
+    for i in bootstrap_peers:
+        if i not in peerList and i not in comm_inst.offlinePeers and not i in gettransports.get() and len(str(i).strip()) > 0:
+            peerList.append(i)
+            keydb.addkeys.add_address(i)
--- a/onionr/communicator/onlinepeers/README.md
+++ b/onionr/communicator/onlinepeers/README.md
@ -0,0 +1,12 @@
+# Online Peers
+
+Manages a pool of peers to perform actions with. Since Onionr does not maintain socket connections, it holds a list of peers.
+
+
+## Files
+
+* \_\_init\_\_.py: exposes some functions to interact with the pool
+* clearofflinepeer.py: Pop the oldest peer in the offline list
+* onlinepeers.py: communicator timer to add new peers to the pool randomly
+* pickonlinepeers.py: returns a random peer from the online pool
+* removeonlinepeer.py: removes a specified peer from the online pool
--- a/onionr/communicator/onlinepeers/init.py
+++ b/onionr/communicator/onlinepeers/init.py
@ -0,0 +1,6 @@
+from . import clearofflinepeer, onlinepeers, pickonlinepeers, removeonlinepeer
+
+clear_offline_peer = clearofflinepeer.clear_offline_peer
+get_online_peers = onlinepeers.get_online_peers
+pick_online_peer = pickonlinepeers.pick_online_peer
+remove_online_peer = removeonlinepeer.remove_online_peer
--- a/onionr/communicator/onlinepeers/clearofflinepeer.py
+++ b/onionr/communicator/onlinepeers/clearofflinepeer.py
@ -0,0 +1,29 @@
+'''
+    Onionr - Private P2P Communication
+
+    clear offline peer in a communicator instance
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import logger
+def clear_offline_peer(comm_inst):
+    '''Removes the longest offline peer to retry later'''
+    try:
+        removed = comm_inst.offlinePeers.pop(0)
+    except IndexError:
+        pass
+    else:
+        logger.debug('Removed ' + removed + ' from offline list, will try them again.')
+    comm_inst.decrementThreadCount('clear_offline_peer')
--- a/onionr/communicator/onlinepeers/onlinepeers.py
+++ b/onionr/communicator/onlinepeers/onlinepeers.py
@ -0,0 +1,45 @@
+'''
+    Onionr - Private P2P Communication
+
+    get online peers in a communicator instance
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import time
+from etc import humanreadabletime
+import logger
+def get_online_peers(comm_inst):
+    '''
+        Manages the comm_inst.onlinePeers attribute list, connects to more peers if we have none connected
+    '''
+    config = comm_inst.config
+    logger.debug('Refreshing peer pool...')
+    maxPeers = int(config.get('peers.max_connect', 10))
+    needed = maxPeers - len(comm_inst.onlinePeers)
+
+    for i in range(needed):
+        if len(comm_inst.onlinePeers) == 0:
+            comm_inst.connectNewPeer(useBootstrap=True)
+        else:
+            comm_inst.connectNewPeer()
+
+        if comm_inst.shutdown:
+            break
+    else:
+        if len(comm_inst.onlinePeers) == 0:
+            logger.debug('Couldn\'t connect to any peers.' + (' Last node seen %s ago.' % humanreadabletime.human_readable_time(time.time() - comm_inst.lastNodeSeen) if not comm_inst.lastNodeSeen is None else ''), terminal=True)
+        else:
+            comm_inst.lastNodeSeen = time.time()
+    comm_inst.decrementThreadCount('get_online_peers')
--- a/onionr/communicator/onlinepeers/pickonlinepeers.py
+++ b/onionr/communicator/onlinepeers/pickonlinepeers.py
@ -0,0 +1,35 @@
+'''
+    Onionr - Private P2P Communication
+
+    pick online peers in a communicator instance
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import secrets
+def pick_online_peer(comm_inst):
+    '''randomly picks peer from pool without bias (using secrets module)'''
+    ret_data = ''
+    while True:
+        peer_length = len(comm_inst.onlinePeers)
+        if peer_length <= 0:
+            break
+        try:
+            # get a random online peer, securely. May get stuck in loop if network is lost or if all peers in pool magically disconnect at once
+            ret_data = comm_inst.onlinePeers[secrets.randbelow(peer_length)]
+        except IndexError:
+            pass
+        else:
+            break
+    return ret_data
--- a/onionr/communicator/onlinepeers/removeonlinepeer.py
+++ b/onionr/communicator/onlinepeers/removeonlinepeer.py
@ -0,0 +1,33 @@
+'''
+    Onionr - Private P2P Communication
+
+    remove an online peer from the pool in a communicator instance
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+def remove_online_peer(comm_inst, peer):
+    '''Remove an online peer'''
+    try:
+        del comm_inst.connectTimes[peer]
+    except KeyError:
+        pass
+    try:
+        del comm_inst.dbTimestamps[peer]
+    except KeyError:
+        pass
+    try:
+        comm_inst.onlinePeers.remove(peer)
+    except ValueError:
+        pass
--- a/onionr/communicator/peeraction.py
+++ b/onionr/communicator/peeraction.py
@ -0,0 +1,55 @@
+'''
+    Onionr - Private P2P Communication
+
+    This file implements logic for performing requests to Onionr peers
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import streamedrequests
+import logger
+from onionrutils import epoch, basicrequests
+from coredb import keydb
+from . import onlinepeers
+
+def peer_action(comm_inst, peer, action, returnHeaders=False, max_resp_size=5242880):
+    '''Perform a get request to a peer'''
+    penalty_score = -10
+    if len(peer) == 0:
+        return False
+    url = 'http://%s/%s' % (peer, action)
+
+    try:
+        ret_data = basicrequests.do_get_request(url, port=comm_inst.proxyPort,
+                                                max_size=max_resp_size)
+    except streamedrequests.exceptions.ResponseLimitReached:
+        logger.warn('Request failed due to max response size being overflowed', terminal=True)
+        ret_data = False
+        penalty_score = -100
+    # if request failed, (error), mark peer offline
+    if ret_data == False: # For some reason "if not" breaks this. Prob has to do with empty string.
+        try:
+            comm_inst.getPeerProfileInstance(peer).addScore(penalty_score)
+            onlinepeers.remove_online_peer(comm_inst, peer)
+            keydb.transportinfo.set_address_info(peer, 'lastConnectAttempt', epoch.get_epoch())
+            if action != 'ping' and not comm_inst.shutdown:
+                logger.warn(f'Lost connection to {peer}', terminal=True)
+                onlinepeers.get_online_peers(comm_inst) # Will only add a new peer to pool if needed
+        except ValueError:
+            pass
+    else:
+        peer_profile = comm_inst.getPeerProfileInstance(peer)
+        peer_profile.update_connect_time()
+        peer_profile.addScore(1)
+    return ret_data # If returnHeaders, returns tuple of data, headers. if not, just data string
--- a/onionr/communicator/uploadqueue/init.py
+++ b/onionr/communicator/uploadqueue/init.py
@ -0,0 +1,60 @@
+"""
+    Onionr - Private P2P Communication
+
+    Class to remember blocks that need to be uploaded and not shared on startup/shutdown
+"""
+"""
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
+import atexit
+import json
+
+import deadsimplekv
+
+import filepaths
+from onionrutils import localcommand
+
+UPLOAD_MEMORY_FILE = filepaths.upload_list
+
+def _add_to_hidden_blocks(cache):
+    for bl in cache:
+        localcommand.local_command('waitforshare/' + bl, post=True)
+
+class UploadQueue:
+    """
+        Saves and loads block upload info from json file
+    """
+
+    def __init__(self, communicator: 'OnionrCommunicatorDaemon'):
+        """Start the UploadQueue object, loading left over uploads into queue 
+        and registering save shutdown function
+        """
+        self.communicator = communicator
+        cache = deadsimplekv.DeadSimpleKV(UPLOAD_MEMORY_FILE)
+        self.store_obj = cache
+        cache: list = cache.get('uploads')
+        if cache == None:
+            cache = []
+        
+        _add_to_hidden_blocks(cache)
+        self.communicator.blocksToUpload.extend(cache)
+
+        atexit.register(self.save)
+
+    def save(self):
+        """Saves to disk on shutdown or if called manually"""
+        bl: list = self.communicator.blocksToUpload
+        self.store_obj.put('uploads', bl)
+        self.store_obj.flush()
--- a/onionr/communicatorutils/README.md
+++ b/onionr/communicatorutils/README.md
@ -0,0 +1,33 @@
+# communicatorutils
+
+The files in this submodule handle various subtasks and utilities for the onionr communicator.
+
+## Files:
+
+announcenode.py: Uses a communicator instance to announce our transport address to connected nodes
+
+connectnewpeers.py: takes a communicator instance and has it connect to as many peers as needed, and/or to a new specified peer.
+
+cooldownpeer.py: randomly selects a connected peer in a communicator and disconnects them for the purpose of security and network balancing.
+
+daemonqueuehandler.py: checks for new commands in the daemon queue and processes them accordingly.
+
+deniableinserts.py: insert fake blocks with the communicator for plausible deniability
+
+downloadblocks.py: iterates a communicator instance's block download queue and attempts to download the blocks from online peers
+
+housekeeping.py: cleans old blocks and forward secrecy keys
+
+lookupadders.py: ask connected peers to share their list of peer transport addresses
+
+lookupblocks.py: lookup new blocks from connected peers from the communicator
+
+netcheck.py: check if the node is online based on communicator status and onion server ping results
+
+onionrcommunicataortimers.py: create a timer for a function to be launched on an interval. Control how many possible instances of a timer may be running a function at once and control if the timer should be ran in a thread or not.
+
+proxypicker.py: returns a string name for the appropriate proxy to be used with a particular peer transport address.
+
+servicecreator.py: iterate connection blocks and create new direct connection servers for them.
+
+uploadblocks.py: iterate a communicator's upload queue and upload the blocks to connected peers
--- a/onionr/communicatorutils/init.py
+++ b/onionr/communicatorutils/init.py
--- a/onionr/communicatorutils/announcenode.py
+++ b/onionr/communicatorutils/announcenode.py
@ -0,0 +1,86 @@
+'''
+    Onionr - Private P2P Communication
+
+    Use a communicator instance to announce our transport address to connected nodes
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import base64
+import onionrproofs, logger
+from etc import onionrvalues
+from onionrutils import basicrequests, bytesconverter
+from utils import gettransports
+from netcontroller import NetController
+from communicator import onlinepeers
+from coredb import keydb
+def announce_node(daemon):
+    '''Announce our node to our peers'''
+    ret_data = False
+    announce_fail = False
+    
+    # Do not let announceCache get too large
+    if len(daemon.announceCache) >= 10000:
+        daemon.announceCache.popitem()
+
+    if daemon.config.get('general.security_level', 0) == 0:
+        # Announce to random online peers
+        for i in daemon.onlinePeers:
+            if not i in daemon.announceCache and not i in daemon.announceProgress:
+                peer = i
+                break
+        else:
+            peer = onlinepeers.pick_online_peer(daemon)
+
+        for x in range(1):
+            try:
+                ourID = gettransports.get()[0]
+            except IndexError:
+                break
+
+            url = 'http://' + peer + '/announce'
+            data = {'node': ourID}
+
+            combinedNodes = ourID + peer
+            if ourID != 1:
+                existingRand = bytesconverter.bytes_to_str(keydb.transportinfo.get_address_info(peer, 'powValue'))
+                # Reset existingRand if it no longer meets the minimum POW
+                if type(existingRand) is type(None) or not existingRand.endswith('0' * onionrvalues.ANNOUNCE_POW):
+                    existingRand = ''
+
+            if peer in daemon.announceCache:
+                data['random'] = daemon.announceCache[peer]
+            elif len(existingRand) > 0:
+                data['random'] = existingRand
+            else:
+                daemon.announceProgress[peer] = True
+                proof = onionrproofs.DataPOW(combinedNodes, minDifficulty=onionrvalues.ANNOUNCE_POW)
+                del daemon.announceProgress[peer]
+                try:
+                    data['random'] = base64.b64encode(proof.waitForResult()[1])
+                except TypeError:
+                    # Happens when we failed to produce a proof
+                    logger.error("Failed to produce a pow for announcing to " + peer)
+                    announce_fail = True
+                else:
+                    daemon.announceCache[peer] = data['random']
+            if not announce_fail:
+                logger.info('Announcing node to ' + url)
+                if basicrequests.do_post_request(url, data, port=daemon.shared_state.get(NetController).socksPort) == 'Success':
+                    logger.info('Successfully introduced node to ' + peer, terminal=True)
+                    ret_data = True
+                    keydb.transportinfo.set_address_info(peer, 'introduced', 1)
+                    keydb.transportinfo.set_address_info(peer, 'powValue', data['random'])
+    daemon.decrementThreadCount('announce_node')
+    return ret_data
--- a/onionr/communicatorutils/connectnewpeers.py
+++ b/onionr/communicatorutils/connectnewpeers.py
@ -0,0 +1,91 @@
+'''
+    Onionr - Private P2P Communication
+
+    Connect a new peer to our communicator instance. Does so randomly if no peer is specified
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import time, sys, secrets
+import onionrexceptions, logger, onionrpeers
+from utils import networkmerger, gettransports
+from onionrutils import stringvalidators, epoch
+from communicator import peeraction, bootstrappeers
+from coredb import keydb
+def connect_new_peer_to_communicator(comm_inst, peer='', useBootstrap=False):
+    config = comm_inst.config
+    retData = False
+    tried = comm_inst.offlinePeers
+    transports = gettransports.get()
+    if peer != '':
+        if stringvalidators.validate_transport(peer):
+            peerList = [peer]
+        else:
+            raise onionrexceptions.InvalidAddress('Will not attempt connection test to invalid address')
+    else:
+        peerList = keydb.listkeys.list_adders()
+
+    mainPeerList = keydb.listkeys.list_adders()
+    peerList = onionrpeers.get_score_sorted_peer_list()
+
+    # If we don't have enough peers connected or random chance, select new peers to try
+    if len(peerList) < 8 or secrets.randbelow(4) == 3:
+        tryingNew = []
+        for x in comm_inst.newPeers:
+            if x not in peerList:
+                peerList.append(x)
+                tryingNew.append(x)
+        for i in tryingNew:
+            comm_inst.newPeers.remove(i)
+    
+    if len(peerList) == 0 or useBootstrap:
+        # Avoid duplicating bootstrap addresses in peerList
+        bootstrappeers.add_bootstrap_list_to_peer_list(comm_inst, peerList)
+
+    for address in peerList:
+        if not config.get('tor.v3onions') and len(address) == 62:
+            continue
+        # Don't connect to our own address
+        if address in transports:
+            continue
+        # Don't connect to invalid address or if its already been tried/connected, or if its cooled down
+        if len(address) == 0 or address in tried or address in comm_inst.onlinePeers or address in comm_inst.cooldownPeer:
+            continue
+        if comm_inst.shutdown:
+            return
+        # Ping a peer,
+        ret = peeraction.peer_action(comm_inst, address, 'ping')
+        if ret == 'pong!':
+            time.sleep(0.1)
+            if address not in mainPeerList:
+                # Add a peer to our list if it isn't already since it successfully connected
+                networkmerger.mergeAdders(address)
+            if address not in comm_inst.onlinePeers:
+                logger.info('Connected to ' + address, terminal=True)
+                comm_inst.onlinePeers.append(address)
+                comm_inst.connectTimes[address] = epoch.get_epoch()
+            retData = address
+
+            # add peer to profile list if they're not in it
+            for profile in comm_inst.peerProfiles:
+                if profile.address == address:
+                    break
+            else:
+                comm_inst.peerProfiles.append(onionrpeers.PeerProfiles(address))
+            break
+        else:
+            # Mark a peer as tried if they failed to respond to ping
+            tried.append(address)
+            logger.debug('Failed to connect to %s: %s ' % (address, ret))
+    return retData
--- a/onionr/communicatorutils/cooldownpeer.py
+++ b/onionr/communicatorutils/cooldownpeer.py
@ -0,0 +1,54 @@
+'''
+    Onionr - Private P2P Communication
+
+    Select a random online peer in a communicator instance and have them "cool down"
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+from onionrutils import epoch
+from communicator import onlinepeers
+def cooldown_peer(comm_inst):
+    '''Randomly add an online peer to cooldown, so we can connect a new one'''
+    config = comm_inst.config
+    onlinePeerAmount = len(comm_inst.onlinePeers)
+    minTime = 300
+    cooldownTime = 600
+    toCool = ''
+    tempConnectTimes = dict(comm_inst.connectTimes)
+
+    # Remove peers from cooldown that have been there long enough
+    tempCooldown = dict(comm_inst.cooldownPeer)
+    for peer in tempCooldown:
+        if (epoch.get_epoch() - tempCooldown[peer]) >= cooldownTime:
+            del comm_inst.cooldownPeer[peer]
+
+    # Cool down a peer, if we have max connections alive for long enough
+    if onlinePeerAmount >= config.get('peers.max_connect', 10, save = True):
+        finding = True
+
+        while finding:
+            try:
+                toCool = min(tempConnectTimes, key=tempConnectTimes.get)
+                if (epoch.get_epoch() - tempConnectTimes[toCool]) < minTime:
+                    del tempConnectTimes[toCool]
+                else:
+                    finding = False
+            except ValueError:
+                break
+        else:
+            onlinepeers.remove_online_peer(comm_inst, toCool)
+            comm_inst.cooldownPeer[toCool] = epoch.get_epoch()
+
+    comm_inst.decrementThreadCount('cooldown_peer')
--- a/onionr/communicatorutils/daemonqueuehandler.py
+++ b/onionr/communicatorutils/daemonqueuehandler.py
@ -0,0 +1,71 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    Handle daemon queue commands in the communicator
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import logger
+import onionrevents as events
+from onionrutils import localcommand
+from coredb import daemonqueue
+import filepaths
+from . import restarttor
+def handle_daemon_commands(comm_inst):
+    cmd = daemonqueue.daemon_queue()
+    response = ''
+    if cmd is not False:
+        events.event('daemon_command', data = {'cmd' : cmd})
+        if cmd[0] == 'shutdown':
+            comm_inst.shutdown = True
+        elif cmd[0] == 'remove_from_insert_list':
+            try:
+                comm_inst.generating_blocks.remove(cmd[1])
+            except ValueError:
+                pass
+        elif cmd[0] == 'announceNode':
+            if len(comm_inst.onlinePeers) > 0:
+                comm_inst.announce(cmd[1])
+            else:
+                logger.debug("No nodes connected. Will not introduce node.")
+        elif cmd[0] == 'runCheck': # deprecated
+            logger.debug('Status check; looks good.')
+            open(filepaths.run_check_file + '.runcheck', 'w+').close()
+        elif cmd[0] == 'connectedPeers':
+            response = '\n'.join(list(comm_inst.onlinePeers)).strip()
+            if response == '':
+                response = 'none'
+        elif cmd[0] == 'localCommand':
+            response = localcommand.local_command(cmd[1])
+        elif cmd[0] == 'clearOffline':
+            comm_inst.offlinePeers = []
+        elif cmd[0] == 'restartTor':
+            restarttor.restart(comm_inst)
+            comm_inst.offlinePeers = []
+        elif cmd[0] == 'pex':
+            for i in comm_inst.timers:
+                if i.timerFunction.__name__ == 'lookupAdders':
+                    i.count = (i.frequency - 1)
+        elif cmd[0] == 'uploadBlock':
+            comm_inst.blocksToUpload.append(cmd[1])
+        else:
+            logger.debug('Received daemon queue command unable to be handled: %s' % (cmd[0],))
+
+        if cmd[0] not in ('', None):
+            if response != '':
+                localcommand.local_command('queueResponseAdd/' + cmd[4], post=True, postData={'data': response})
+        response = ''
+
+    comm_inst.decrementThreadCount('handle_daemon_commands')
--- a/onionr/communicatorutils/deniableinserts.py
+++ b/onionr/communicatorutils/deniableinserts.py
@ -0,0 +1,32 @@
+'''
+    Onionr - Private P2P Communication
+
+    Use the communicator to insert fake mail messages
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import secrets
+from etc import onionrvalues
+import onionrblocks
+def insert_deniable_block(comm_inst):
+    '''Insert a fake block in order to make it more difficult to track real blocks'''
+    fakePeer = ''
+    chance = 10
+    if secrets.randbelow(chance) == (chance - 1):
+        # This assumes on the libsodium primitives to have key-privacy
+        fakePeer = onionrvalues.DENIABLE_PEER_ADDRESS
+        data = secrets.token_hex(secrets.randbelow(1024) + 1)
+        onionrblocks.insert(data, header='pm', encryptType='asym', asymPeer=fakePeer, disableForward=True, meta={'subject': 'foo'})
+    comm_inst.decrementThreadCount('insert_deniable_block')
--- a/onionr/communicatorutils/downloadblocks/init.py
+++ b/onionr/communicatorutils/downloadblocks/init.py
@ -0,0 +1,132 @@
+'''
+    Onionr - Private P2P Communication
+
+    Download blocks using the communicator instance
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import communicator, onionrexceptions
+import logger, onionrpeers
+from onionrutils import blockmetadata, stringvalidators, validatemetadata
+from coredb import blockmetadb
+from . import shoulddownload
+from communicator import peeraction, onlinepeers
+import onionrcrypto, onionrstorage, onionrblacklist, storagecounter
+def download_blocks_from_communicator(comm_inst):
+    '''Use Onionr communicator instance to download blocks in the communicator's queue'''
+    assert isinstance(comm_inst, communicator.OnionrCommunicatorDaemon)
+    blacklist = onionrblacklist.OnionrBlackList()
+    storage_counter = storagecounter.StorageCounter()
+    LOG_SKIP_COUNT = 50 # for how many iterations we skip logging the counter
+    count: int = 0
+    metadata_validation_result: bool = False
+    # Iterate the block queue in the communicator
+    for blockHash in list(comm_inst.blockQueue):
+        count += 1
+        if len(comm_inst.onlinePeers) == 0:
+            break
+        triedQueuePeers = [] # List of peers we've tried for a block
+        try:
+            blockPeers = list(comm_inst.blockQueue[blockHash])
+        except KeyError:
+            blockPeers = []
+        removeFromQueue = True
+
+        if not shoulddownload.should_download(comm_inst, blockHash):
+            continue
+
+        if comm_inst.shutdown or not comm_inst.isOnline or storage_counter.is_full():
+            # Exit loop if shutting down or offline, or disk allocation reached
+            break
+        # Do not download blocks being downloaded
+        if blockHash in comm_inst.currentDownloading:
+            #logger.debug('Already downloading block %s...' % blockHash)
+            continue
+
+        comm_inst.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block
+        if len(blockPeers) == 0:
+            peerUsed = onlinepeers.pick_online_peer(comm_inst)
+        else:
+            blockPeers = onionrcrypto.cryptoutils.random_shuffle(blockPeers)
+            peerUsed = blockPeers.pop(0)
+
+        if not comm_inst.shutdown and peerUsed.strip() != '':
+            logger.info("Attempting to download %s from %s..." % (blockHash[:12], peerUsed))
+        content = peeraction.peer_action(comm_inst, peerUsed, 'getdata/' + blockHash, max_resp_size=3000000) # block content from random peer (includes metadata)
+
+        if content != False and len(content) > 0:
+            try:
+                content = content.encode()
+            except AttributeError:
+                pass
+
+            realHash = onionrcrypto.hashers.sha3_hash(content)
+            try:
+                realHash = realHash.decode() # bytes on some versions for some reason
+            except AttributeError:
+                pass
+            if realHash == blockHash:
+                #content = content.decode() # decode here because sha3Hash needs bytes above
+                metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
+                metadata = metas[0]
+                try:
+                    metadata_validation_result = validatemetadata.validate_metadata(metadata, metas[2])
+                except onionrexceptions.DataExists:
+                    metadata_validation_result = False
+                if metadata_validation_result: # check if metadata is valid, and verify nonce
+                    if onionrcrypto.cryptoutils.verify_POW(content): # check if POW is enough/correct
+                        logger.info('Attempting to save block %s...' % blockHash[:12])
+                        try:
+                            onionrstorage.set_data(content)
+                        except onionrexceptions.DataExists:
+                            logger.warn('Data is already set for %s ' % (blockHash,))
+                        except onionrexceptions.DiskAllocationReached:
+                            logger.error('Reached disk allocation allowance, cannot save block %s.' % (blockHash,))
+                            removeFromQueue = False
+                        else:
+                            blockmetadb.add_to_block_DB(blockHash, dataSaved=True) # add block to meta db
+                            blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database
+                    else:
+                        logger.warn('POW failed for block %s.' % (blockHash,))
+                else:
+                    if blacklist.inBlacklist(realHash):
+                        logger.warn('Block %s is blacklisted.' % (realHash,))
+                    else:
+                        logger.warn('Metadata for block %s is invalid.' % (blockHash,))
+                        blacklist.addToDB(blockHash)
+            else:
+                # if block didn't meet expected hash
+                tempHash = onionrcrypto.hashers.sha3_hash(content) # lazy hack, TODO use var
+                try:
+                    tempHash = tempHash.decode()
+                except AttributeError:
+                    pass
+                # Punish peer for sharing invalid block (not always malicious, but is bad regardless)
+                onionrpeers.PeerProfiles(peerUsed).addScore(-50)
+                if tempHash != 'ed55e34cb828232d6c14da0479709bfa10a0923dca2b380496e6b2ed4f7a0253':
+                    # Dumb hack for 404 response from peer. Don't log it if 404 since its likely not malicious or a critical error.
+                    logger.warn('Block hash validation failed for ' + blockHash + ' got ' + tempHash)
+                else:
+                    removeFromQueue = False # Don't remove from queue if 404
+            if removeFromQueue:
+                try:
+                    del comm_inst.blockQueue[blockHash] # remove from block queue both if success or false
+                    if count == LOG_SKIP_COUNT:
+                        logger.info('%s blocks remaining in queue' % [len(comm_inst.blockQueue)], terminal=True)
+                        count = 0
+                except KeyError:
+                    pass
+        comm_inst.currentDownloading.remove(blockHash)
+    comm_inst.decrementThreadCount('getBlocks')
--- a/onionr/communicatorutils/downloadblocks/shoulddownload.py
+++ b/onionr/communicatorutils/downloadblocks/shoulddownload.py
@ -0,0 +1,36 @@
+'''
+    Onionr - Private P2P Communication
+
+    Check if a block should be downloaded (if we already have it or its blacklisted or not)
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+from coredb import blockmetadb
+import onionrblacklist
+def should_download(comm_inst, block_hash):
+    blacklist = onionrblacklist.OnionrBlackList()
+    ret_data = True
+    if block_hash in blockmetadb.get_block_list(): # Dont download block we have
+        ret_data = False
+    else:
+        if blacklist.inBlacklist(block_hash): # Dont download blacklisted block
+            ret_data = False
+    if ret_data is False:
+        # Remove block from communicator queue if it shouldnt be downloaded
+        try:
+            del comm_inst.blockQueue[block_hash]
+        except KeyError:
+            pass
+    return ret_data
--- a/onionr/communicatorutils/housekeeping.py
+++ b/onionr/communicatorutils/housekeeping.py
@ -0,0 +1,75 @@
+'''
+    Onionr - Private P2P Communication
+
+    Cleanup old Onionr blocks and forward secrecy keys using the communicator. Ran from a timer usually
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sqlite3
+import logger
+from onionrusers import onionrusers
+from onionrutils import epoch
+from coredb import blockmetadb, dbfiles
+import onionrstorage
+from onionrstorage import removeblock
+import onionrblacklist
+
+def __remove_from_upload(comm_inst, block_hash: str):
+    try:
+        comm_inst.blocksToUpload.remove(block_hash)
+    except ValueError:
+        pass
+
+def clean_old_blocks(comm_inst):
+    '''Delete old blocks if our disk allocation is full/near full, and also expired blocks'''
+    blacklist = onionrblacklist.OnionrBlackList()
+    # Delete expired blocks
+    for bHash in blockmetadb.expiredblocks.get_expired_blocks():
+        blacklist.addToDB(bHash)
+        removeblock.remove_block(bHash)
+        onionrstorage.deleteBlock(bHash)
+        __remove_from_upload(comm_inst, bHash)
+        logger.info('Deleted block: %s' % (bHash,))
+
+    while comm_inst.storage_counter.is_full():
+        oldest = blockmetadb.get_block_list()[0]
+        blacklist.addToDB(oldest)
+        removeblock.remove_block(oldest)
+        onionrstorage.deleteBlock(oldest)
+        __remove_from_upload.remove(comm_inst, oldest)
+        logger.info('Deleted block: %s' % (oldest,))
+
+    comm_inst.decrementThreadCount('clean_old_blocks')
+
+def clean_keys(comm_inst):
+    '''Delete expired forward secrecy keys'''
+    conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=10)
+    c = conn.cursor()
+    time = epoch.get_epoch()
+    deleteKeys = []
+
+    for entry in c.execute("SELECT * FROM forwardKeys WHERE expire <= ?", (time,)):
+        logger.debug('Forward key: %s' % entry[1])
+        deleteKeys.append(entry[1])
+
+    for key in deleteKeys:
+        logger.debug('Deleting forward key %s' % key)
+        c.execute("DELETE from forwardKeys where forwardKey = ?", (key,))
+    conn.commit()
+    conn.close()
+
+    onionrusers.deleteExpiredKeys()
+
+    comm_inst.decrementThreadCount('clean_keys')
--- a/onionr/communicatorutils/lookupadders.py
+++ b/onionr/communicatorutils/lookupadders.py
@ -0,0 +1,55 @@
+'''
+    Onionr - Private P2P Communication
+
+    Lookup new peer transport addresses using the communicator
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import logger
+from onionrutils import stringvalidators
+from communicator import peeraction, onlinepeers
+from utils import gettransports
+def lookup_new_peer_transports_with_communicator(comm_inst):
+    logger.info('Looking up new addresses...')
+    tryAmount = 1
+    newPeers = []
+    transports = gettransports.get()
+
+    for i in range(tryAmount):
+        # Download new peer address list from random online peers
+        if len(newPeers) > 10000:
+            # Don't get new peers if we have too many queued up
+            break
+        peer = onlinepeers.pick_online_peer(comm_inst)
+        newAdders = peeraction.peer_action(comm_inst, peer, action='pex')
+        try:
+            newPeers = newAdders.split(',')
+        except AttributeError:
+            pass
+    else:
+        # Validate new peers are good format and not already in queue
+        invalid = []
+        for x in newPeers:
+            x = x.strip()
+            if not stringvalidators.validate_transport(x) or x in comm_inst.newPeers or x in transports:
+                # avoid adding if its our address
+                invalid.append(x)
+        for x in invalid:
+            try:
+                newPeers.remove(x)
+            except ValueError:
+                pass
+        comm_inst.newPeers.extend(newPeers)
+    comm_inst.decrementThreadCount('lookup_new_peer_transports_with_communicator')
--- a/onionr/communicatorutils/lookupblocks.py
+++ b/onionr/communicatorutils/lookupblocks.py
@ -0,0 +1,94 @@
+'''
+    Onionr - Private P2P Communication
+
+    Lookup new blocks with the communicator using a random connected peer
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import logger, onionrproofs
+from onionrutils import stringvalidators, epoch
+from communicator import peeraction, onlinepeers
+from coredb import blockmetadb
+from utils import reconstructhash
+import onionrblacklist
+blacklist = onionrblacklist.OnionrBlackList()
+def lookup_blocks_from_communicator(comm_inst):
+    logger.info('Looking up new blocks')
+    tryAmount = 2
+    newBlocks = ''
+    existingBlocks = blockmetadb.get_block_list() # List of existing saved blocks
+    triedPeers = [] # list of peers we've tried this time around
+    maxBacklog = 1560 # Max amount of *new* block hashes to have already in queue, to avoid memory exhaustion
+    lastLookupTime = 0 # Last time we looked up a particular peer's list
+    new_block_count = 0
+    for i in range(tryAmount):
+        listLookupCommand = 'getblocklist' # This is defined here to reset it each time
+        if len(comm_inst.blockQueue) >= maxBacklog:
+            break
+        if not comm_inst.isOnline:
+            break
+        # check if disk allocation is used
+        if comm_inst.storage_counter.is_full():
+            logger.debug('Not looking up new blocks due to maximum amount of allowed disk space used')
+            break
+        peer = onlinepeers.pick_online_peer(comm_inst) # select random online peer
+        # if we've already tried all the online peers this time around, stop
+        if peer in triedPeers:
+            if len(comm_inst.onlinePeers) == len(triedPeers):
+                break
+            else:
+                continue
+        triedPeers.append(peer)
+
+        # Get the last time we looked up a peer's stamp to only fetch blocks since then.
+        # Saved in memory only for privacy reasons
+        try:
+            lastLookupTime = comm_inst.dbTimestamps[peer]
+        except KeyError:
+            lastLookupTime = 0
+        else:
+            listLookupCommand += '?date=%s' % (lastLookupTime,)
+        try:
+            newBlocks = peeraction.peer_action(comm_inst, peer, listLookupCommand) # get list of new block hashes
+        except Exception as error:
+            logger.warn('Could not get new blocks from %s.' % peer, error = error)
+            newBlocks = False
+        else:
+            comm_inst.dbTimestamps[peer] = epoch.get_rounded_epoch(roundS=60)
+        if newBlocks != False:
+            # if request was a success
+            for i in newBlocks.split('\n'):
+                if stringvalidators.validate_hash(i):
+                    i = reconstructhash.reconstruct_hash(i)
+                    # if newline seperated string is valid hash
+                    if not i in existingBlocks:
+                        # if block does not exist on disk and is not already in block queue
+                        if i not in comm_inst.blockQueue:
+                            if onionrproofs.hashMeetsDifficulty(i) and not blacklist.inBlacklist(i):
+                                if len(comm_inst.blockQueue) <= 1000000:
+                                    comm_inst.blockQueue[i] = [peer] # add blocks to download queue
+                                    new_block_count += 1
+                        else:
+                            if peer not in comm_inst.blockQueue[i]:
+                                if len(comm_inst.blockQueue[i]) < 10:
+                                    comm_inst.blockQueue[i].append(peer)
+    if new_block_count > 0:
+        block_string = ""
+        if new_block_count > 1:
+            block_string = "s"
+        logger.info('Discovered %s new block%s' % (new_block_count, block_string), terminal=True)
+        comm_inst.download_blocks_timer.count = int(comm_inst.download_blocks_timer.frequency * 0.99)
+    comm_inst.decrementThreadCount('lookup_blocks_from_communicator')
+    return
--- a/onionr/communicatorutils/netcheck.py
+++ b/onionr/communicatorutils/netcheck.py
@ -0,0 +1,43 @@
+'''
+    Onionr - Private P2P Communication
+
+    Determine if our node is able to use Tor based on the status of a communicator instance
+    and the result of pinging onion http servers
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import logger
+from utils import netutils
+from onionrutils import localcommand, epoch
+from . import restarttor
+def net_check(comm_inst):
+    '''Check if we are connected to the internet or not when we can't connect to any peers'''
+    rec = False # for detecting if we have received incoming connections recently
+    if len(comm_inst.onlinePeers) == 0:
+        try:
+            if (epoch.get_epoch() - int(localcommand.local_command('/lastconnect'))) <= 60:
+                comm_inst.isOnline = True
+                rec = True
+        except ValueError:
+            pass
+        if not rec and not netutils.checkNetwork(torPort=comm_inst.proxyPort):
+            if not comm_inst.shutdown:
+                logger.warn('Network check failed, are you connected to the Internet, and is Tor working?', terminal=True)
+                restarttor.restart(comm_inst)
+                comm_inst.offlinePeers = []
+            comm_inst.isOnline = False
+        else:
+            comm_inst.isOnline = True
+    comm_inst.decrementThreadCount('net_check')
--- a/onionr/communicatorutils/onionrcommunicatortimers.py
+++ b/onionr/communicatorutils/onionrcommunicatortimers.py
@ -0,0 +1,79 @@
+'''
+    Onionr - Private P2P Communication
+
+    This file contains timer control for the communicator
+'''
+from __future__ import annotations  # thank you python, very cool
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+import uuid
+import threading
+
+import onionrexceptions, logger
+
+from typing import TYPE_CHECKING
+from typing import Callable, NewType, Iterable
+if TYPE_CHECKING:
+    from communicator import OnionrCommunicatorDaemon
+
+CallFreqSeconds = NewType('CallFreqSeconds', int)
+
+class OnionrCommunicatorTimers:
+    def __init__(self, daemon_inst: OnionrCommunicatorDaemon, 
+                timer_function: Callable, frequency: CallFreqSeconds, 
+                make_thread:bool=True, thread_amount:int=1, max_threads:int=5, 
+                requires_peer:bool=False, my_args:Iterable=[]):
+        self.timer_function = timer_function
+        self.frequency = frequency
+        self.thread_amount = thread_amount
+        self.make_thread = make_thread
+        self.requires_peer = requires_peer
+        self.daemon_inst = daemon_inst
+        self.max_threads = max_threads
+        self.args = my_args
+
+        self.daemon_inst.timers.append(self)
+        self.count = 0
+
+    def processTimer(self):
+
+        # mark how many instances of a thread we have (decremented at thread end)
+        try:
+            self.daemon_inst.threadCounts[self.timer_function.__name__]
+        except KeyError:
+            self.daemon_inst.threadCounts[self.timer_function.__name__] = 0
+
+        # execute thread if it is time, and we are not missing *required* online peer
+        if self.count == self.frequency and not self.daemon_inst.shutdown:
+            try:
+                if self.requires_peer and len(self.daemon_inst.onlinePeers) == 0:
+                    raise onionrexceptions.OnlinePeerNeeded
+            except onionrexceptions.OnlinePeerNeeded:
+                return
+            else:
+                if self.make_thread:
+                    for i in range(self.thread_amount):
+                        if self.daemon_inst.threadCounts[self.timer_function.__name__] >= self.max_threads:
+                            logger.debug('%s is currently using the maximum number of threads, not starting another.' % self.timer_function.__name__, terminal=True)
+                        else:
+                            self.daemon_inst.threadCounts[self.timer_function.__name__] += 1
+                            newThread = threading.Thread(target=self.timer_function, args=self.args, daemon=True, 
+                                                        name=self.timer_function.__name__ + ' - ' + str(uuid.uuid4()))
+                            newThread.start()
+                else:
+                    self.timer_function()
+            self.count = -1 # negative 1 because its incremented at bottom
+        self.count += 1
--- a/onionr/communicatorutils/proxypicker.py
+++ b/onionr/communicatorutils/proxypicker.py
@ -0,0 +1,26 @@
+'''
+    Onionr - Private P2P Communication
+
+    Just picks a proxy to use based on a peer's address
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+def pick_proxy(peer_address):
+    if peer_address.endswith('.onion'):
+        return 'tor'
+    elif peer_address.endswith('.i2p'):
+        return 'i2p'
+    raise ValueError("Peer address was not string ending with acceptable value")
--- a/onionr/communicatorutils/restarttor.py
+++ b/onionr/communicatorutils/restarttor.py
@ -0,0 +1,5 @@
+import netcontroller
+def restart(comm_inst):
+    net = comm_inst.shared_state.get(netcontroller.NetController)
+    net.killTor()
+    net.startTor()
--- a/onionr/communicatorutils/servicecreator.py
+++ b/onionr/communicatorutils/servicecreator.py
@ -0,0 +1,44 @@
+'''
+    Onionr - Private P2P Communication
+
+    Creates an onionr direct connection service by scanning all connection blocks
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import communicator, onionrblockapi
+import logger
+from onionrutils import stringvalidators, bytesconverter
+from coredb import blockmetadb
+from onionrservices import server_exists
+def service_creator(daemon):
+    assert isinstance(daemon, communicator.OnionrCommunicatorDaemon)
+    
+    # Find socket connection blocks
+    # TODO cache blocks and only look at recently received ones
+    con_blocks = blockmetadb.get_blocks_by_type('con')
+    for b in con_blocks:
+        if not b in daemon.active_services:
+            bl = onionrblockapi.Block(b, decrypt=True)
+            bs = bytesconverter.bytes_to_str(bl.bcontent) + '.onion'
+            if server_exists(bl.signer):
+                continue
+            if stringvalidators.validate_pub_key(bl.signer) and stringvalidators.validate_transport(bs):
+                signer = bytesconverter.bytes_to_str(bl.signer)
+                daemon.active_services.append(b)
+                daemon.active_services.append(signer)
+                if not daemon.services.create_server(signer, bs, daemon):
+                    daemon.active_services.remove(b)
+                    daemon.active_services.remove(signer)
+    daemon.decrementThreadCount('service_creator')
--- a/onionr/communicatorutils/uploadblocks/init.py
+++ b/onionr/communicatorutils/uploadblocks/init.py
@ -0,0 +1,92 @@
+'''
+    Onionr - Private P2P Communication
+
+    Upload blocks in the upload queue to peers from the communicator
+'''
+from __future__ import annotations
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+from typing import Union, TYPE_CHECKING
+import threading
+
+import logger
+from communicatorutils import proxypicker
+import onionrexceptions
+import onionrblockapi as block
+from onionrutils import localcommand, stringvalidators, basicrequests
+from communicator import onlinepeers
+import onionrcrypto
+from . import sessionmanager
+
+def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
+    """Accepts a communicator instance and uploads blocks from its upload queue"""
+    """when inserting a block, we try to upload
+     it to a few peers to add some deniability & increase functionality"""
+    TIMER_NAME = "upload_blocks_from_communicator"
+
+    session_manager: sessionmanager.BlockUploadSessionManager = comm_inst.shared_state.get(sessionmanager.BlockUploadSessionManager)
+    triedPeers = []
+    finishedUploads = []
+    comm_inst.blocksToUpload = onionrcrypto.cryptoutils.random_shuffle(comm_inst.blocksToUpload)
+    if len(comm_inst.blocksToUpload) != 0:
+        for bl in comm_inst.blocksToUpload:
+            if not stringvalidators.validate_hash(bl):
+                logger.warn('Requested to upload invalid block', terminal=True)
+                comm_inst.decrementThreadCount(TIMER_NAME)
+                return
+            session = session_manager.add_session(bl)
+            for i in range(min(len(comm_inst.onlinePeers), 6)):
+                peer = onlinepeers.pick_online_peer(comm_inst)
+                try:
+                    session.peer_exists[peer]
+                    continue
+                except KeyError:
+                    pass
+                try:
+                    if session.peer_fails[peer] > 3: continue
+                except KeyError:
+                    pass
+                if peer in triedPeers: continue
+                triedPeers.append(peer)
+                url = f'http://{peer}/upload'
+                try:
+                    data = block.Block(bl).getRaw()
+                except onionrexceptions.NoDataAvailable:
+                    finishedUploads.append(bl)
+                    break
+                proxyType = proxypicker.pick_proxy(peer)
+                logger.info(f"Uploading block {bl[:8]} to {peer}", terminal=True)
+                resp = basicrequests.do_post_request(url, data=data, proxyType=proxyType, content_type='application/octet-stream')
+                if not resp == False:
+                    if resp == 'success':
+                        session.success()
+                        session.peer_exists[peer] = True
+                    elif resp == 'exists':
+                        session.success()
+                        session.peer_exists[peer] = True
+                    else:
+                        session.fail()
+                        session.fail_peer(peer)
+                        comm_inst.getPeerProfileInstance(peer).addScore(-5)
+                        logger.warn(f'Failed to upload {bl[:8]}, reason: {resp[:15]}', terminal=True)
+                else:
+                    session.fail()
+        session_manager.clean_session()
+    for x in finishedUploads:
+        try:
+            comm_inst.blocksToUpload.remove(x)
+        except ValueError:
+            pass
+    comm_inst.decrementThreadCount(TIMER_NAME)
--- a/onionr/communicatorutils/uploadblocks/session.py
+++ b/onionr/communicatorutils/uploadblocks/session.py
@ -0,0 +1,54 @@
+"""
+    Onionr - Private P2P Communication
+
+    Virtual upload "sessions" for blocks
+"""
+from __future__ import annotations
+"""
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+from typing import Union
+
+from onionrutils import stringvalidators
+from onionrutils import bytesconverter
+from onionrutils import epoch
+from utils import reconstructhash
+
+class UploadSession:
+    """Manages statistics for an Onionr block upload session
+    
+    accepting a block hash (incl. unpadded) as an argument""" 
+    def __init__(self, block_hash: Union[str, bytes]):
+        block_hash = bytesconverter.bytes_to_str(block_hash)
+        block_hash = reconstructhash.reconstruct_hash(block_hash)
+        if not stringvalidators.validate_hash(block_hash): raise ValueError
+
+        self.start_time = epoch.get_epoch()
+        self.block_hash = reconstructhash.deconstruct_hash(block_hash)
+        self.total_fail_count: int = 0
+        self.total_success_count: int = 0
+        self.peer_fails = {}
+        self.peer_exists = {}
+    
+    def fail_peer(self, peer):
+        try:
+            self.peer_fails[peer] += 1
+        except KeyError:
+            self.peer_fails[peer] = 0
+    
+    def fail(self):
+        self.total_fail_count += 1
+    
+    def success(self):
+        self.total_success_count += 1
--- a/onionr/communicatorutils/uploadblocks/sessionmanager.py
+++ b/onionr/communicatorutils/uploadblocks/sessionmanager.py
@ -0,0 +1,85 @@
+"""
+    Onionr - Private P2P Communication
+
+    Manager for upload 'sessions'
+"""
+from __future__ import annotations
+"""
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+from typing import Iterable, Union
+
+from onionrutils import bytesconverter
+from onionrutils import localcommand
+from etc import onionrvalues
+from etc import waitforsetvar
+from utils import reconstructhash
+
+from . import session
+
+class BlockUploadSessionManager:
+    """Holds block UploadSession instances. Optionally accepts iterable of sessions to added on init
+
+    Arguments: old_session: iterable of old UploadSession objects"""
+    def __init__(self, old_sessions:Iterable=None):
+        #self._too_many: TooMany = None
+        if old_sessions is None:
+            self.sessions = []
+        else:
+            self.sessions = old_session
+    
+    def add_session(self, session_or_block: Union(str, bytes, session.UploadSession))->session.UploadSession:
+        """Create (or add existing) block upload session from a str/bytes block hex hash, existing UploadSession"""
+        if isinstance(session_or_block, session.UploadSession): 
+            if not session_or_block in self.sessions:
+                self.sessions.append(session_or_block)
+            return session_or_block
+        try:
+            return self.get_session(session_or_block)
+        except KeyError:
+            pass
+        # convert bytes hash to str
+        if isinstance(session_or_block, bytes): session_or_block = bytesconverter.bytes_to_str(session_or_block)
+        # intentionally not elif
+        if isinstance(session_or_block, str):
+            new_session = session.UploadSession(session_or_block)
+            self.sessions.append(new_session)
+            return new_session
+
+    def get_session(self, block_hash: Union(str, bytes))->session.UploadSession:
+        block_hash = reconstructhash.deconstruct_hash(bytesconverter.bytes_to_str(block_hash))
+        for session in self.sessions:
+            if session.block_hash == block_hash: return session
+        raise KeyError
+
+    def clean_session(self, specific_session: Union[str, UploadSession]=None):
+        comm_inst: OnionrCommunicatorDaemon = self._too_many.get_by_string("OnionrCommunicatorDaemon")
+        sessions_to_delete = []
+        if comm_inst.getUptime() < 120: return
+        for session in self.sessions:
+            if (session.total_success_count / len(comm_inst.onlinePeers)) >= onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT:
+                sessions_to_delete.append(session)
+        for session in sessions_to_delete:
+            self.sessions.remove(session)
+            # TODO cleanup to one round of search
+            # Remove the blocks from the sessions, upload list, and waitforshare list
+            try:
+                comm_inst.blocksToUpload.remove(reconstructhash.reconstruct_hash(session.block_hash))
+            except ValueError:
+                pass
+            try:
+                comm_inst.blocksToUpload.remove(session.block_hash)
+            except ValueError:
+                pass
+            localcommand.local_command('waitforshare/{session.block_hash}')
--- a/src/config/init.py
+++ b/src/config/init.py
@ -1,35 +1,33 @@
-"""Onionr - Private P2P Communication.
+'''
+    Onionr - Private P2P Communication

-This file deals with configuration management.
-"""
-import os
-from json import JSONDecodeError
+    This file deals with configuration management.
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.

-import ujson as json
-from logger import log as logging
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+import os, json, logger
 import filepaths

-"""
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-"""
-
 _configfile = filepaths.config_file
 _config = {}

-
 def get(key, default = None, save = False):
-    """Gets the key from configuration, or returns `default`"""
+    '''
+        Gets the key from configuration, or returns `default`
+    '''

    key = str(key).split('.')
    data = _config
@ -41,20 +39,20 @@ def get(key, default = None, save = False):
            return default
        data = data[item]

-    if last not in data:
+    if not last in data:
        if save:
            set(key, default, savefile = True)
        return default

    return data[last]

-
 def set(key, value = None, savefile = False):
-    """Sets the key in configuration to `value`"""
+    '''
+        Sets the key in configuration to `value`
+    '''

    global _config

-    whole_key = key
    key = str(key).split('.')
    data = _config

@ -73,7 +71,6 @@ def set(key, value = None, savefile = False):
    if savefile:
        save()

-
 def is_set(key):
    key = str(key).split('.')
    data = _config
@ -90,53 +87,60 @@ def is_set(key):

    return True

-
 def check():
-    """Checks if the configuration file exists, creates it if not"""
+    '''
+        Checks if the configuration file exists, creates it if not
+    '''

    if not os.path.exists(os.path.dirname(get_config_file())):
        os.makedirs(os.path.dirname(get_config_file()))

-
 def save():
-    """Saves the configuration data to the configuration file"""
+    '''
+        Saves the configuration data to the configuration file
+    '''

    check()
    try:
        with open(get_config_file(), 'w', encoding="utf8") as configfile:
            json.dump(get_config(), configfile, indent=2)
-    except JSONDecodeError:
-        logging.warn('Failed to write to configuration file.')
-
+    except json.JSONDecodeError:
+        logger.warn('Failed to write to configuration file.')

 def reload():
-    """Reloads the configuration data in memory from the file"""
+    '''
+        Reloads the configuration data in memory from the file
+    '''
    check()
    try:
        with open(get_config_file(), 'r', encoding="utf8") as configfile:
            set_config(json.loads(configfile.read()))
-    except (FileNotFoundError, JSONDecodeError) as e:
+    except (FileNotFoundError, json.JSONDecodeError) as e:
        pass
-        #logging.debug('Failed to parse configuration file.')
-
+        #logger.debug('Failed to parse configuration file.')

 def get_config():
-    """Gets the entire configuration as an array"""
+    '''
+        Gets the entire configuration as an array
+    '''
    return _config

-
 def set_config(config):
-    """Sets the configuration to the array in arguments"""
+    '''
+        Sets the configuration to the array in arguments
+    '''
    global _config
    _config = config

-
 def get_config_file():
-    """Returns the absolute path to the configuration file"""
+    '''
+        Returns the absolute path to the configuration file
+    '''
    return _configfile

-
 def set_config_file(configfile):
-    """Sets the path to the configuration file."""
+    '''
+        Sets the path to the configuration file
+    '''
    global _configfile
    _configfile = os.abs.abspath(configfile)
--- a/onionr/coredb/init.py
+++ b/onionr/coredb/init.py
@ -0,0 +1 @@
+from . import keydb, blockmetadb, daemonqueue
--- a/onionr/coredb/blockmetadb/init.py
+++ b/onionr/coredb/blockmetadb/init.py
@ -0,0 +1,80 @@
+'''
+    Onionr - Private P2P Communication
+
+    This module works with information relating to blocks stored on the node
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sqlite3
+from . import expiredblocks, updateblockinfo, add
+from .. import dbfiles
+update_block_info = updateblockinfo.update_block_info
+add_to_block_DB = add.add_to_block_DB
+def get_block_list(dateRec = None, unsaved = False):
+    '''
+        Get list of our blocks
+    '''
+    if dateRec == None:
+        dateRec = 0
+
+    conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30)
+    c = conn.cursor()
+
+    execute = 'SELECT hash FROM hashes WHERE dateReceived >= ? ORDER BY dateReceived ASC;'
+    args = (dateRec,)
+    rows = list()
+    for row in c.execute(execute, args):
+        for i in row:
+            rows.append(i)
+    conn.close()
+    return rows
+
+def get_block_date(blockHash):
+    '''
+        Returns the date a block was received
+    '''
+
+    conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30)
+    c = conn.cursor()
+
+    execute = 'SELECT dateReceived FROM hashes WHERE hash=?;'
+    args = (blockHash,)
+    for row in c.execute(execute, args):
+        for i in row:
+            return int(i)
+    conn.close()
+    return None
+
+def get_blocks_by_type(blockType, orderDate=True):
+    '''
+        Returns a list of blocks by the type
+    '''
+
+    conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30)
+    c = conn.cursor()
+
+    if orderDate:
+        execute = 'SELECT hash FROM hashes WHERE dataType=? ORDER BY dateReceived;'
+    else:
+        execute = 'SELECT hash FROM hashes WHERE dataType=?;'
+
+    args = (blockType,)
+    rows = list()
+
+    for row in c.execute(execute, args):
+        for i in row:
+            rows.append(i)
+    conn.close()
+    return rows
--- a/onionr/coredb/blockmetadb/add.py
+++ b/onionr/coredb/blockmetadb/add.py
@ -0,0 +1,42 @@
+'''
+    Onionr - Private P2P Communication
+
+    Add an entry to the block metadata database
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import os, sqlite3, secrets
+from onionrutils import epoch, blockmetadata
+from .. import dbfiles
+def add_to_block_DB(newHash, selfInsert=False, dataSaved=False):
+    '''
+        Add a hash value to the block db
+
+        Should be in hex format!
+    '''
+
+    if blockmetadata.has_block(newHash):
+        return
+    conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30)
+    c = conn.cursor()
+    currentTime = epoch.get_epoch() + secrets.randbelow(301)
+    if selfInsert or dataSaved:
+        selfInsert = 1
+    else:
+        selfInsert = 0
+    data = (newHash, currentTime, '', selfInsert)
+    c.execute('INSERT INTO hashes (hash, dateReceived, dataType, dataSaved) VALUES(?, ?, ?, ?);', data)
+    conn.commit()
+    conn.close()
--- a/onionr/coredb/blockmetadb/expiredblocks.py
+++ b/onionr/coredb/blockmetadb/expiredblocks.py
@ -0,0 +1,37 @@
+'''
+    Onionr - Private P2P Communication
+
+    Get a list of expired blocks still stored
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sqlite3
+from onionrutils import epoch
+from .. import dbfiles
+def get_expired_blocks():
+    '''Returns a list of expired blocks'''
+    conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30)
+    c = conn.cursor()
+    date = int(epoch.get_epoch())
+
+    compiled = (date,)
+    execute = 'SELECT hash FROM hashes WHERE expire <= ? ORDER BY dateReceived;'
+
+    rows = list()
+    for row in c.execute(execute, compiled):
+        for i in row:
+            rows.append(i)
+    conn.close()
+    return rows
--- a/onionr/coredb/blockmetadb/updateblockinfo.py
+++ b/onionr/coredb/blockmetadb/updateblockinfo.py
@ -0,0 +1,49 @@
+'''
+    Onionr - Private P2P Communication
+
+    Update block information in the metadata database by a field name
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sqlite3
+from .. import dbfiles
+def update_block_info(hash, key, data):
+    '''
+        sets info associated with a block
+
+        hash         - the hash of a block
+        dateReceived - the date the block was recieved, not necessarily when it was created
+        decrypted    - if we can successfully decrypt the block (does not describe its current state)
+        dataType     - data type of the block
+        dataFound    - if the data has been found for the block
+        dataSaved    - if the data has been saved for the block
+        sig    - optional signature by the author (not optional if author is specified)
+        author       - multi-round partial sha3-256 hash of authors public key
+        dateClaimed  - timestamp claimed inside the block, only as trustworthy as the block author is
+        expire       - expire date for a block
+    '''
+    if key not in ('dateReceived', 'decrypted', 'dataType', 'dataFound', 
+                  'dataSaved', 'sig', 'author', 'dateClaimed', 'expire'):
+        raise ValueError('Key must be in the allowed list')
+
+    conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30)
+    c = conn.cursor()
+    args = (data, hash)
+    # Unfortunately, not really possible
+    c.execute("UPDATE hashes SET " + key + " = ? where hash = ?;", args)
+    conn.commit()
+    conn.close()
+
+    return True
--- a/onionr/coredb/daemonqueue/init.py
+++ b/onionr/coredb/daemonqueue/init.py
@ -0,0 +1,92 @@
+'''
+    Onionr - Private P2P Communication
+
+    Write and read the daemon queue, which is how messages are passed into the onionr daemon in a more
+    direct way than the http api
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sqlite3, os
+import onionrevents as events
+from onionrutils import localcommand, epoch
+from .. import dbfiles
+import dbcreator
+
+def daemon_queue()->str:
+    '''
+        Gives commands to the communication proccess/daemon by reading an sqlite3 database
+
+        This function intended to be used by the client. Queue to exchange data between "client" and server.
+    '''
+
+    retData = False
+    if not os.path.exists(dbfiles.daemon_queue_db):
+        dbcreator.createDaemonDB()
+    else:
+        conn = sqlite3.connect(dbfiles.daemon_queue_db, timeout=30)
+        c = conn.cursor()
+        try:
+            for row in c.execute('SELECT command, data, date, min(ID), responseID FROM commands group by id'):
+                retData = row
+                break
+        except sqlite3.OperationalError:
+            dbcreator.createDaemonDB()
+        else:
+            if retData != False:
+                c.execute('DELETE FROM commands WHERE id=?;', (retData[3],))
+        conn.commit()
+        conn.close()
+
+    return retData
+
+def daemon_queue_add(command: str, data='', responseID: str =''):
+    '''
+        Add a command to the daemon queue, used by the communication daemon (communicator.py)
+    '''
+
+    retData = True
+
+    date = epoch.get_epoch()
+    conn = sqlite3.connect(dbfiles.daemon_queue_db, timeout=30)
+    c = conn.cursor()
+    t = (command, data, date, responseID)
+    try:
+        c.execute('INSERT INTO commands (command, data, date, responseID) VALUES(?, ?, ?, ?)', t)
+        conn.commit()
+    except sqlite3.OperationalError:
+        retData = False
+        daemon_queue()
+    conn.close()
+    return retData
+
+def daemon_queue_get_response(responseID=''):
+    '''
+        Get a response sent by communicator to the API, by requesting to the API
+    '''
+    if len(responseID) == 0: raise ValueError('ResponseID should not be empty')
+    resp = localcommand.local_command(dbfiles.daemon_queue_db, 'queueResponse/' + responseID)
+    return resp
+
+def clear_daemon_queue():
+    '''
+        Clear the daemon queue (somewhat dangerous)
+    '''
+    conn = sqlite3.connect(dbfiles.daemon_queue_db, timeout=30)
+    c = conn.cursor()
+
+    c.execute('DELETE FROM commands;')
+    conn.commit()
+
+    conn.close()
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Kevin Froman	7db3e20867	uploads now have sessions	2019-09-19 18:34:16 -05:00
Kevin Froman	15d04d9f2f	added python info to version command	2019-09-19 15:16:31 -05:00
Kevin Froman	af8d7a3e96	work on fixing uploads	2019-09-18 15:56:29 -05:00
				`@ -0,0 +1 @@`
				`from . import keydb, blockmetadb, daemonqueue`