Add PKGBUILD

.dockerignore

@@ -1,6 +1,4 @@
 onionr/data/**/*
 onionr/data
+RUN-WINDOWS.bat
 MY-RUN.sh
-Dockerfile
-.dockerignore
-.git

.env

@@ -1 +0,0 @@
-PYTHONPATH=./venv/bin/python310:./src/

.github/FUNDING.yml

@@ -1,4 +0,0 @@
-# These are supported funding model platforms
-
-ko_fi: beardogkf
-

.github/stale.yml

@@ -1,7 +0,0 @@
-# Number of days of inactivity before an issue becomes stale
-daysUntilStale: 0
-# Number of days of inactivity before a stale issue is closed
-daysUntilClose: 0
-# Label to use when marking an issue as stale
-staleLabel: wontfix
-closeComment: false

.github/workflows/greetings.yml

@@ -1,13 +0,0 @@
-name: Greetings
-
-on: [pull_request, issues]
-
-jobs:
-  greeting:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/first-interaction@v1
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        issue-message: 'Message that will be displayed on users'' first issue'
-        pr-message: 'Hi, please make the PR at https://gitlab.com/beardog/Onionr'' first pr'

.gitignore

@@ -1,44 +1,32 @@
 __pycache__/
-src/data/config.ini
-src/data/*.db
-src/data-old/*
-src/data*
-src/tor
-src/tor.exe
-src/testdata
-src/*.pyc
-src/*.log
-src/data/hs/hostname
-src/data/*
-src/data-backup/*
+onionr/data/config.ini
+onionr/data/*.db
+onionr/data-old/*
+onionr/data*
+onionr/testdata
+onionr/*.pyc
+onionr/*.log
+onionr/data/hs/hostname
+onionr/data/*
+onionr/data-backup/*
+onionr/gnupg/*
 run.sh
-src/.onionr-lock
-daemon-true.txt
-.vscode/tags
-.vscode/settings.json
-
-venv*
-src/fs*
-src/tmp/*
-testdata/*
-*.dll
-*.exe
-
-.mypy_cache/
-
-dist/*
+onionr/data-encrypted.dat
+onionr/.onionr-lock
+core
+.vscode/*
+venv/*
+onionr/fs*
 
 # log files
 output.log
 *.log
-src/output.log
-src/*.log
-src/data/output.log
-src/data/*.log
+onionr/output.log
+onionr/*.log
+onionr/data/output.log
+onionr/data/*.log
 
 # package files
 onionr-*.pkg.tar.gz
 pkg/
-spawnnodes.py
-
-static-data/tor-node-list.dat
+src/

.gitlab-ci.yml

@@ -1,11 +1,7 @@
 test:
-  image: python
+  image: ubuntu:bionic
   script:
   - apt-get update -qy
-  - apt-get install -y tor
-  - apt-get install -y firefox-esr
-  - pip3 install --require-hashes -r requirements.txt
-  - pip3 install --require-hashes -r requirements-dev.txt
-  - pip3 install --require-hashes -r requirements-notifications.txt
-  - python3 --version
+  - apt-get install -y python3-pip tor
+  - pip3 install -r requirements.txt
   - make test

.vscode/launch.json

@@ -1,23 +0,0 @@
-{
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "name": "Onionr: init",
-            "type": "python",
-            "request": "launch",
-            "program": "${workspaceFolder}/src/__init__.py",
-            "cwd": "${workspaceFolder}/src/",
-            "args" : ["start"]
-        },
-        {
-            "name": "Python: Current File",
-            "type": "python",
-            "request": "launch",
-            "program": "${file}",
-            "cwd": "${workspaceFolder}/src/"
-        }
-    ]
-}

.vscode/tasks.json

@@ -1,58 +0,0 @@
-{
-    // See https://go.microsoft.com/fwlink/?LinkId=733558
-    // for the documentation about the tasks.json format
-    "version": "2.0.0",
-    "tasks": [
-      {
-        "label": "Make new test",
-        "type": "shell",
-        "command": "cp tests/test_template.py tests/new-test.py",
-        "group": "test",
-        "presentation": {
-          "reveal": "always",
-          "panel": "new"
-        }
-      },
-      {
-        "label": "Run tests",
-        "type": "shell",
-        "command": "source venv/bin/activate; make test",
-        "group": "test",
-        "presentation": {
-          "reveal": "always",
-          "panel": "new"
-        }
-      },
-      {
-        "label": "Run test by name",
-        "type": "shell",
-        "command": "source venv/bin/activate; scripts/run-unit-test-by-name.py",
-        "group": "test",
-        "presentation": {
-          "reveal": "always",
-          "panel": "new"
-        }
-      },
-      {
-        "label": "Enable dev config",
-        "type": "process",
-        "command": "scripts/enable-dev-config.py",
-        "group": "dev",
-        "problemMatcher": []
-      },
-      {
-        "label": "Disable dev config",
-        "type": "process",
-        "command": "scripts/disable-dev-config.py",
-        "group": "dev",
-        "problemMatcher": []
-      },
-      {
-        "label": "Format default config",
-        "type": "process",
-        "command": "scripts/pretty-default-config.py",
-        "group": "dev",
-        "problemMatcher": []
-      }
-    ]
-  }

CHANGELOG.md

@@ -1,48 +0,0 @@
-# Changelog
-
-All notable changes to this project will be documented in this file.
-
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
-and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-
-## [8.0.1] - 2020-12-22
-
-* Fix subprocess in 3.9x with dumb hack
-* Dependency bumps
-
-## [8.0.0] - 2020-12-04
-
-* Decrease PoW until better implementation is made
-
-
-## [7.2.0] - 2020-12-03
-
-* Purge blocks not meeting current pow on startup
-* Check block POW before LAN sync
-* WSL fixes
-
-## [7.1.0] - 2020-11-23
-
-* Check for ownership of existing dirs in createdirs, this prevents the rare edge case where a user might use a home directory in a location an attacker could write (allowing arbitrary code execution via plugins). This was already partially mitigated by the chmod of the home directory in any case, but this further fixes the issue.
-
-## [7.0.0] - 2020-11-22
-
-* Removed communicator timers
-* Removed direct connections and chat (these will be either plugins or separate programs/processes in the future)
-
-
-## [5.1.0] - 2020-09-07
-
-* Moved plugin web files to be in the plugin folder to reduce staticfiles blueprint coupling
-* Added basic sidebar on index page
-* Many bug fixes
-
-
-## [5.0.1] - 2020-08-08
-
-* bumped deadsimplekv to 0.3.2
-* bumped urllib3 to 1.25.10
-
-## [5.0.0] - 2020-07-23
-
-- Removed single-process POW support (was only needed on Windows)

CODE_OF_CONDUCT.md

@@ -40,7 +40,7 @@ Project maintainers who do not follow or enforce the Code of Conduct in good fai
 
 ## Attribution
 
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [https://contributor-covenant.org/version/1/4][version]
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]
 
-[homepage]: https://contributor-covenant.org
-[version]: https://contributor-covenant.org/version/1/4/
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

CONTRIBUTING.md

@@ -1,33 +1,38 @@
 # Contributing to Onionr
 
-One of the great things about open source projects is that they allow for anyone to contribute to the project.
+One of the great things about open source projects is that they allow for many people to contribute to the project.
 
 This file serves to provide guidelines on how to successfully contribute to Onionr.
 
 ## Code of Conduct
 
-Contributors in project-related spaces/contexts are expected to follow the [Code of Conduct](https://github.com/beardog108/onionr/blob/master/CODE_OF_CONDUCT.md)
+See our [Code of Conduct](https://github.com/beardog108/onionr/blob/master/CODE_OF_CONDUCT.md)
 
 ## Reporting Bugs
 
 Bugs can be reported using GitLab issues. Please try to see if an issue is already opened for a particular thing.
 
-See the [issue template](ISSUE_TEMPLATE.md) for what to include when reporting a bug.
+Please provide the following information when reporting a bug:
+
+* Operating system
+* Python version
+* Onionr version
+* Onionr logs or output with errors, any possible relevant information.
+* A description of what you were doing before the bug was experienced
+* Screenshots can often be helpful, but videos are rarely helpful.
 
 If a bug is a security issue, please contact us privately.
 
-Please be patient. Onionr is a free open source project maintained by volunteers in their free time.
+And most importantly, please be patient. Onionr is an open source project done by volunteers.
 
 ## Asking Questions
 
-If you need help with Onionr, you can contact the devs (be polite and remember this is a volunteer-driven project).
-
-Do your best to use good english.
+If you need help with Onionr, you can contact the devs (be polite and remember this is a volunteer-driven non-profit project).
 
 ## Contributing Code
 
 For any non-trivial changes, please get in touch with us first to discuss your plans.
 
-Please try to use a similar coding style as the project. We like PEP-8 despite being lazy with it in the past.
+Please try to use a similar coding style as the project.
 
 **Thanks for contributing to Onionr!**

Dockerfile

@@ -1,31 +1,28 @@
-FROM python:3.10
-EXPOSE 8080
+FROM ubuntu:bionic
 
-USER root
-
-RUN mkdir /app
-WORKDIR /app
-
-ENV ONIONR_DOCKER=true
+#Base settings
+ENV HOME /root
 
 #Install needed packages
-RUN apt-get update && apt-get install -y tor locales
+RUN apt update && apt install -y python3 python3-dev python3-pip tor locales nano sqlite3
 
 RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
     locale-gen
-ENV LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8
+ENV LANG en_US.UTF-8  
+ENV LANGUAGE en_US:en  
+ENV LC_ALL en_US.UTF-8  
 
-ADD ./requirements-x86-all-plugins.txt /app/requirements.txt
-RUN pip3 install --require-hashes -r requirements.txt
+WORKDIR /srv/
+ADD ./requirements.txt /srv/requirements.txt
+RUN pip3 install -r requirements.txt
 
+WORKDIR /root/
 #Add Onionr source
-COPY . /app/
+COPY . /root/
+VOLUME /root/data/
 
-VOLUME /app/data/
+#Set upstart command
+CMD bash
 
-#Default to running as nonprivileged user
-RUN chmod g=u -R /app
-USER 1000
-ENV HOME=/app
-
-CMD ["bash", "./onionr.sh", "start"]
+#Expose ports
+EXPOSE 8080

ISSUE_TEMPLATE.md

@@ -5,9 +5,8 @@
 # Steps to Reproduce
 
 # Version Information
-
-* Onionr: 
-* OS: 
-* Python: 
-* Tor: 
-* CPU:
+Onionr: 
+OS: 
+Python: 
+Tor: 
+I2P: 

LICENSE.txt

@@ -1,3 +1,6 @@
+The Onionr logo was created by [Anhar Ismail](https://github.com/anharismail) under the [Creative Commons Attribution 4.0 International License](https://creativecommons.org/licenses/by/4.0/). 
+
+
                     GNU GENERAL PUBLIC LICENSE
                        Version 3, 29 June 2007
 

Makefile

@@ -25,16 +25,16 @@ test:
 
 soft-reset:
 	@echo "Soft-resetting Onionr..."
-	./onionr.sh soft-reset
+	rm -f onionr/$(ONIONR_HOME)/blocks/*.dat onionr/data/*.db onionr/$(ONIONR_HOME)/block-nonces.dat | true > /dev/null 2>&1
 	@./onionr.sh version | grep -v "Failed" --color=always
 
 reset:
 	@echo "Hard-resetting Onionr..."
-	rm -rf $(ONIONR_HOME)/ | true > /dev/null 2>&1
+	rm -rf onionr/$(ONIONR_HOME)/ | true > /dev/null 2>&1
 	cd onionr/static-data/www/ui/; rm -rf ./dist; python compile.py
 	#@./onionr.sh version | grep -v "Failed" --color=always
 
 plugins-reset:
 	@echo "Resetting plugins..."
-	@./onionr.sh reset-plugins | true > /dev/null 2>&1
+	rm -rf onionr/$(ONIONR_HOME)/plugins/ | true > /dev/null 2>&1
 	@./onionr.sh version | grep -v "Failed" --color=always

PKGBUILD

@@ -1,5 +1,5 @@
-# Maintainer: Aaron Esau <aur@aaronesau.com>
-pkgname="onionr-git"
+# Maintainer: Kevin Froman <contact@onionr.net>
+pkgname="onionr"
 pkgver="0.1"
 pkgrel="1"
 conflicts=("onionr2")

README.md

@@ -5,166 +5,105 @@
 </p>
 
 <p align="center">
-    Privacy Respecting Communication Network 📡
-</p>
-<p align="center">
-    WIP anonymous social platform, mail, file sharing and marketplace
+    Anonymous P2P storage network 🕵️
 </p>
 
-<img src="https://img.shields.io/badge/License-aGPLv3-yellow"> <img src='https://img.shields.io/badge/python%20version%20%F0%9F%90%8D-3.10+-blue'>
+(***pre-alpha & experimental, not well tested or easy to use yet***)
 
-<a href='https://twitter.com/onionrnet'><img src='https://img.shields.io/twitter/follow/onionrnet?style=social'></a> - [Discord](https://discord.gg/DVF2bEAzrt) - Matrix: #onionr:amorgan.xyz
+[![Open Source Love](https://badges.frapsoft.com/os/v3/open-source.png?v=103)](https://github.com/ellerbrock/open-source-badges/)
+<img src='https://gitlab.com/beardog/Onionr/badges/master/build.svg'>
 
-|                                      |                            |                            |
-| -----------                          | -----------                | -----------                |
-| [Install](#install-and-run-on-linux) | [Features](#main-features) | [Screenshots](#screenshots)|
-| [Docs](#documentation)               | [Get involved](#help-out)  | [Onionr.net](https://onionr.net/)/[.onion](http://onionrbak72t5zhbzuey2fdkpczlvhowgcpqc6uoyrd3uxztzxwz5cyd.onion/)                  |
+<hr>
 
+**The main repository for this software is at https://gitlab.com/beardog/Onionr/**
 
----
+# Summary
 
-**The main repository for this software is at https://gitlab.com/beardog/onionr/**
+Onionr is a decentralized, peer-to-peer data storage network, designed to be anonymous and resistant to (meta)data analysis and spam/disruption.
 
-***Note that this README reflects the state of the rewrite, and not the original alpha network***
+Onionr stores data in independent packages referred to as 'blocks'. The blocks are synced to all other nodes in the network. Blocks and user IDs cannot be easily proven to have been created by particular nodes (only inferred). Even if there is enough evidence to believe a particular node created a block, nodes still operate behind Tor or I2P and as such are not trivially known to be at a particular IP address.
 
-Mirrors [Gitea](https://git.voidnet.tech/kev/onionr)
-
-
-Onionr ("Onion Relay") is a decentralized/distributed peer-to-peer communication network, designed to be anonymous and resistant to (meta)data analysis, spam, and corruption.
-
-Onionr gives the individual the ability to speak freely, without fear of surveillance and censorship.
-
----
-
-# Onionr internals
-
-## Blocks
-
-At the core, Onionr is an anonymous Distributed Hash Table (DHT) syncing prepackaged blocks using a simple Gossip protocol with Dandelion++ as an overlay network on top of Tor and I2P.
-
-Onionr stores data in independent packages referred to as 'blocks'. The blocks are distributed to all nodes, but are not required to be stored. Blocks and user IDs cannot be easily proven to have been created by a particular user. Even if Dandelion++ is defeated and there is enough evidence to believe that a specific node is linked to a block's creation, nodes still operate behind Tor or I2P and as such cannot be trivially unmasked. Anonymity is achieved by a stateless network, with no given indication of what node a block originates from. In fact, since one is not required to participate in routing or storage to insert a message, blocks often do not originate from any identifiable node, similar to how Bitcoin transactions do not necessarily originate from a wallet directly associated with a node.
-
-## Onionr Gossip
-
-Onionr works via epidemic/gossip style routing, with message delivery taking roughly log<sub>C</sub>(N) cycles where C is the number of nodes to send a message to each cycle and N is the number of connected nodes. So a network of 100 million nodes can deliver messages in a few minutes even with high packet loss and malfunctioning nodes.
-
-Through Dandelion++ message forwarding and key privacy, it is intended to be nigh impossible to discover the identity of a message creator or recipient. Via long-term traffic analysis, a well funded adversary may discover the most probable node(s) to be creating a set of related blocks, however doing so would only lead them to a node behind Tor. As the first node that a block appears on is almost always not the creator of the block, there is plausible deniability regarding the true creator of the block.
-
-Users are identified by ed25519/curve25519 public keys, which can be used to sign blocks or send encrypted data.
+Users are identified by ed25519 public keys, which can be used to sign blocks or send encrypted data.
 
 Onionr can be used for mail, as a social network, instant messenger, file sharing software, or for encrypted group discussion.
 
-Since Onionr is technically just a data format, any routing scheme can  technically be used to pass messages.
+![Tor stinks slide image](docs/tor-stinks-02.png)
 
-The whitepaper is available [here](docs/whitepaper.md).
-
----
-
-# Main Features
+## Main Features
 
 * [X] 🌐 Fully p2p/decentralized, no trackers or other single points of failure
+* [X] 🔒 End to end encryption of user data
+* [X] 📢 Optional non-encrypted blocks, useful for blog posts or public file sharing
+* [X] 💻 Easy API system for integration to websites
 * [X] 🕵️ Metadata analysis resistance and anonymity
 * [X] 📡 Transport agnosticism (no internet required)
 
+**Onionr API and functionality is subject to non-backwards compatible change during pre-alpha development**
 
-# Roadmap
+# Screenshots
 
-See [ROADMAP.md](ROADMAP.md)
+<img alt='Node statistics page screenshot' src='docs/onionr-1.png' width=600>
 
-# Documentation
+Node statistics
 
-More docs coming soon.
+<img alt='Friend/contact manager screenshot' src='docs/onionr-2.png' width=600>
 
+Friend/contact manager
+
+<img alt='Encrypted, metadata-masking mail application screenshot' src='docs/onionr-3.png' width=600>
+
+Encrypted, metadata-masking mail application.
 
 # Install and Run on Linux
 
-The following applies to Ubuntu Bionic. Other distributions may have different package or command names.
+The following applies to Ubuntu Bionic. Other distros may have different package or command names.
 
-Master may be unstable, you should use the latest release tag. (checkout via git: `$ git checkout release-latest`)
-
-`$ sudo apt install python3-pip python3-dev tor`
-
-* Have python3.10, python3-pip, Tor (daemon, not browser) installed. python3-dev is recommended.
-* You may need build-essentials or the equivalent of your platform
-* Clone the git repo: `$ git clone https://gitlab.com/beardog/onionr --tags`
+* Have python3.6+, python3-pip, Tor (daemon, not browser) installed (python3-dev recommended)
+* Clone the git repo: `$ git clone https://gitlab.com/beardog/onionr`
 * cd into install direction: `$ cd onionr/`
-* Install the Python dependencies ([virtualenv strongly recommended](https://virtualenv.pypa.io/en/stable/userguide/)): `$ pip3 install --require-hashes -r requirements-x86-all-plugins.txt`
+* Install the Python dependencies ([virtualenv strongly recommended](https://virtualenv.pypa.io/en/stable/userguide/)): `$ pip3 install --require-hashes -r requirements.txt`
 
-Require-hashes is suggested for supply-chain security but is optional. The hashes are not correct for ARM machines. If you are just running a node or want a bare-bones install you can use requirements-base-x86.txt and selectively install the requirements.txt files in static-data/official-plugins/ subdirectories
+(--require-hashes is intended to prevent exploitation via compromise of Pypi/CA certificates)
 
-## Run Onionr
+## Help out
 
-* Run Onionr normally:  `$ ./onionr.sh start`
-* Run Onionr in background as daemon: `$ ./start-daemon.sh`
-* Gracefully stop Onionr from CLI `$ ./onionr.sh stop`
-
-
-# Contact/Community
-
-* Email: onionr [ at ] voidnet.tech
-* Twitter: [@onionrnet](https://twitter.com/onionrnet)
-* Matrix: #onionr:amorgan.xyz
-* Discord: https://discord.gg/DVF2bEAzrt (Discord is bad for freedom and privacy, this is only provided for convenience)
-
-# Help out
-
-Everyone is welcome to contribute. Help is wanted for the following:
+Everyone is welcome to help out. Help is wanted for the following:
 
 * Development (Get in touch first)
     * Creation of a shared lib for use from other languages and faster proof-of-work
     * Android and IOS development
-    * Mac support (testers needed)
-    * Bug fixes and development of new features
+    * Windows and Mac support (already partially supported, testers needed)
+    * General bug fixes and development of new features
 * Testing
-* Translations/localizations
 * UI/UX design
 * Running stable nodes
 * Security review/audit
-* I2P support
+* Automatic I2P setup
 
-## Watch the talk from BSidesPDX 2019
+Contribute money:
 
-<a href="https://www.youtube.com/watch?v=mrULtmSkKxg">
-<img src="docs/talk.png" alt="improving anonymous networking talk link" width="600">
-</a>
+Donating at least $5 gets you cool Onionr stickers. Get in touch if you want them.
 
-## Contribute money:
+Bitcoin: [1onion55FXzm6h8KQw3zFw2igpHcV7LPq](bitcoin:1onion55FXzm6h8KQw3zFw2igpHcV7LPq) (Contact us for privacy coins like Monero)
 
-Donating at least $3 gets you cool Onionr stickers. Get in touch if you want them.
+USD (Card/Paypal): [Ko-Fi](https://www.ko-fi.com/beardogkf)
 
-![sticker](docs/sticker.png)
+Note: probably not tax deductible
 
+## Contact
 
-* Bitcoin: [bc1qpayme9rlpkch0qp3r79lvm5racr7t6llauwfmg](bitcoin:bc1qpayme9rlpkch0qp3r79lvm5racr7t6llauwfmg) (Contact us for a unique address or for other coins)
+beardog [ at ] mailbox.org
 
-* Monero: 4B5BA24d1P3R5aWEpkGY5TP7buJJcn2aSGBVRQCHhpiahxeB4aWsu15XwmuTjC6VF62NApZeJGTS248RMVECP8aW73Uj2ax
-
-* USD (Card/Paypal (no account required)): [Ko-Fi](https://www.ko-fi.com/beardogkf)
-
-* Sign up for [privacy.com (refferal link)](https://privacy.com/join/FNNDF) to protect your personal information when contributing or shopping elsewhere, we both get $5 USD.
-
-Note: not tax deductible
-
-# Security
-
-Onionr is alpha software. This means it is unstable, probably insecure, and experimental.
-
-No matter how good Onionr and other software gets, there will always be ways for clever or well-funded adversaries to break your security.
-
-Onionr does not protect your identity if you associate your user ID with your name either on Onionr or elsewhere.
-
-*Do not rely on Onionr or any other software to hold up if your life or liberty are at stake.*
-
-### Licenses and Branding
-
-Onionr is published under the GNU GPL v3 license, except for the logo.
+## Disclaimer
 
 The Tor Project and I2P developers do not own, create, or endorse this project, and are not otherwise involved.
 
 Tor is a trademark for the Tor Project. We do not own it.
 
-## Onionr Logo
+The 'open source badge' is by Maik Ellerbrock and is licensed under a Creative Commons Attribution 4.0 International License.
 
-The Onionr logo was created by [Anhar Ismail](https://github.com/anharismail) under the [Creative Commons Attribution 4.0 International License](https://creativecommons.org/licenses/by/4.0/).
+## Logo
 
-If you modify and redistribute our code ("forking"), please use a different logo and project name to avoid confusion. Please do not use the project name or logo in a way that makes it seem like we endorse you without our permission.
+The Onionr logo was created by [Anhar Ismail](https://github.com/anharismail) under the [Creative Commons Attribution 4.0 International License](https://creativecommons.org/licenses/by/4.0/). 
+
+If you modify and redistribute our code ("forking"), please use a different logo and project name to avoid confusion. Please do not use our logo in a way that makes it seem like we endorse you without permission.

ROADMAP.md

@@ -1,46 +0,0 @@
-# Onionr Roadmap
-
-
-## Taraxacum Release (9.0)
-
-
-* [ X ] Implement new block format with verifiable delay function
-
-* [ X ] Implement overhauled gossip system with dandelion++
-
-    The new system is separated from the underlying networks used and makes it much easier to implement new transport methods. Dandelion++ is also better than the old block mixing we did.
-
-
-## Misc
-
-* [  ] Spruce up documentation
-* [  ] Restore LAN transport
-* [  ] Restore webUI as a plugin
-* [  ] Restore static site/file sharing plugin
-* [  ] Restore/reimplement mail plugin
-* [  ] Restore/reimplement friends plugin
-* [  ] Refresh test suite
-* [  ] Revamped key/encrypted messaging (encrypted blocks)
-
-
-
-## Web of trust release (~10.0)
-
-To facilitate the below market plugin/application, Onionr will need a web of trust system.
-
-* [ ] Web of trust plugin or module
-
-
-## Market Plugin Release (~10.1)
-
-The Onionr team believes the Monero community is currently lacking a good p2p market, and as such we hope to build a solution using Onionr as a transport. This may be a separate project and as opposed to a plugin.
-
-
-* A new marketplace plugin will be developed with the following *tenative* features:
-
-* [ ] Account management
-* [ ] Monero management
-* [ ] Store front discovery and search
-* [ ] Product listing search
-* [ ] User reviews
-

SECURITY.md

@@ -1,39 +0,0 @@
-# Security Policy
-
-We welcome responsible and constructive security review.
-
-# Scope
-
-The Onionr software and any nodes you control are within scope.
-
-Avoid social engineering, volume-based denial of service and disrupting or harming the Onionr network. Do not attempt to exploit any machines/servers you do not own or otherwise have permission to do so.
-
-The following exploits are of particular interest:
-
-* Arbitrary code execution
-* API authentication bypass (such as accessing local API from public interface)
-* Deanonymization:
-  * Easily associating public keys with server addresses
-  * Discovering true server IPs when behind Tor/I2P (aside from Tor/i2p-level attacks)
-  * Easily discovering which nodes are the block creator
-* XSS, CSRF, clickjacking, DNS rebinding
-* Timing attacks against the local http server that are exploitable from a browser ([see blog post](https://www.chaoswebs.net/blog/timebleed-breaking-privacy-with-a-simple-timing-attack.html))
-* Discovering direct connection servers as a non participant.
-* Cryptography/protocol issues
-* Denying nodes access to the network by segmenting them out with Sybil nodes
-
-We do not consider non-network based same-machine attacks to be very significant, but we are still willing to listen.
-
-# Rewards
-
-Onionr is a student-owned hobby project, resources are not available for large rewards.
-
-Stickers or other small rewards are available. We reserve the right to refuse rewards for any reason.
-
-Public recognition can be given upon request.
-
-# Contact
-
-Email: beardog [ at ] mailbox.org
-
-PGP (optional): F61A 4DBB 0B3D F172 1F65  0EDF 0D41 4D0F E405 B63B

base-requirements.in

@@ -1,8 +0,0 @@
-PyNaCl==1.5.0
-psutil==5.9.1
-filenuke==0.0.0
-ujson==5.4.0
-cffi==1.15.1
-onionrblocks==7.0.0
-ordered-set==4.1.0
-json-rpc==1.13.0

docs/README.md

@@ -1,21 +0,0 @@
-<img src="onionr-logo.png" width="200px">
-
-# Documentation
-
-The Onionr [whitepaper](whitepaper.md) is the best place to start both for users and developers.
-
-## User Documentation
-
-* [Installation](usage/install.md)
-* [First steps](usage/firststeps.md)
-* [Using Onionr Mail](usage/mail.md)
-* [Using Onionr web pages](usage/pages.md)
-* [Staying safe/anonymous](usage/safety.md)
-
-## Developer Documentation
-
-* [Development environment setup](dev/setup.md)
-* [Technical overview](dev/overview.md)
-* [Project layout](dev/layout.md)
-* [Plugin development guide](dev/plugins.md)
-* [Auto generated API docs (HTML)](html/onionr/index.html)

docs/api.md

@@ -0,0 +1,2 @@
+HTTP API
+TODO

docs/dev/daemon-events.md

@@ -1,9 +0,0 @@
-# DaemonEvents
-
-For things that need to be processed by the daemon
-
-Observer pattern
-
-Register listeners dynamically per event
-
-Spawn new greenlets

docs/dev/error-codes.md

@@ -1,4 +0,0 @@
-# Exit codes
-
-10: run with no command
-3: command does not exist

docs/dev/generating-requirements.txt.md

@@ -1,19 +0,0 @@
-# Generating requirements.txt
-
-To generate a requirements.txt file, install pip-tools from pip
-
-Onionr requirements files should have hashes to prevent backdooring by the pypi server.
-
-Put your package versions in requirements.in like normal. Child dependencies are usually not necessary:
-
-```
-requests==0.1.1
-flask==0.1.1
-```
-
-Then generate the requirements.txt:
-
-`$ pip-compile requirements.in --generate-hashes -o requirements.txt`
-
-
-Your requirements.txt will have hash-pinned requirements of all dependencies and child dependencies.

docs/dev/running-tests.md

@@ -1,44 +0,0 @@
-# Running Onionr tests
-
-Onionr has four types of tests:
-
-* unittests
-* integration tests
-* selenium tests (web tests)
-* runtime-tests
-
-
-## unittests
-
-Onionr uses Python's built in unittest module. These tests are located in tests/ (top level)
-
-Run all tests with `$ make test`, which will also run integration tests.
-
-Please note that one unittest tests if runtime-tests have passed recently. This is simply a forceful reminder to run those tests as well.
-
-You can also run a single unittest in a loop by using the script scripts/run-unit-test-by-name.py
-
-## integration tests
-
-These tests are pretty basic and test on stdout of Onionr commands.
-
-They are also run from `$ make test`
-
-The runtime-tests do most of the actual integration testing.
-
-## selenium tests
-
-These are browser automation tests to test if the UI is working as how it should for a user.
-
-There's only a couple and they're incomplete, so they can be ignored for now (test manually)
-
-## runtime-tests
-
-These are important. They look into the Onionr client Flask app when Onionr daemon is running and test a bunch of things.
-
-If you do it a lot you should make your own Onionr network (disable official bootstrap)
-
-You run this while the daemon is running (probably should make sure onboarding is done), with `$ onionr.sh runtime-test`
-
-It's necessary to do this before running `$ make test` for unittesting
-

docs/dev/security-mechanisms.md

@@ -1,43 +0,0 @@
-# Onionr Security Mechanisms
-
-
-## bigbrother 👁️
-
-Bigbrother is a cheeky module that uses Python3.8+ sys auditing events to log and/or block certain sensitive events.
-
-It has a little overhead, so one can disable it in config in general.security_auditing
-
-[ChaosWebs.net/blog/preventing-arbitrary-code-execution-in-python38-with-auditing.html](https://chaoswebs.net/blog/preventing-arbitrary-code-execution-in-python38-with-auditing.html)
-
-### Threat model
-
-It is intended to log bugs leaking private file system information, block+log network leaks, and block+log eval-like arbitrary code execution. It is not intended to block malicious browser scripts or malicious Python plugins. It cannot work with subprocesses that do not activate the module.
-
-It's not intended to be bulletproof by any means, but it helps.
-
-### What big brother does
-
-* Disk access checks for disk access outside. Only logs, does not block
-* Network leaks. (Non Tor/LAN) Blocks and logs
-* Arbitrary code execution: logs and blocks non-whitelisted bytecode importing/compiling and subprocesses.
-
-
-## Sybil attacks
-
-As with any decentralized network, sybil nodes could collude to spy or cause mayhem. Due to the gossip nature of Onionr, sybil nodes would have a hard time fully stopping the network. In terms of spying, they could not conclusively prove the origin of messages due to the multiple transport nature of the network and layering behind Tor/etc.
-
-## Tor configuration
-
-When managed by Onionr, Tor has a control port password that gets stored in Onionr config.
-
-Tor is also configured to reject requests to non-onion services, which helps to stop redirect based denial of service attacks.
-
-## Web security
-
-Onionr secures both it's main web APIs with anti-dns-rebinding logic, which validates the host header used in connections to it. This is to prevent exfiltration of data and side channel deanonymization.
-
-Onionr secures the client API with a token that must be passed in most requests, with the exception of static API files. This is to prevent CSRF and side channel deanonymization.
-
-Onionr binds most services to random loopback addresses to reduce all cross-site web attacks, including discovery of Onionr on a computer from a normal website. This is not supported on Mac because Mac does not support non 'typical' loopback addresses.
-
-Onionr has a strict content-security-policy, rejecting all non-localhost requests and denying inline scripts and similar insecure sources.

docs/dev/selected-papers.md

@@ -1,9 +0,0 @@
-# Interesting papers related to Onionr development
-
-A paper being listed here is not end-all-be-all endorsement of every detail inside.
-
-* [Epidemic Routing for Partially-Connected Ad Hoc Networks](https://web.archive.org/web/20200208074703/http://issg.cs.duke.edu/epidemic/epidemic.pdf)
-* [Freenet: A distibuted decentralized information storage and retrieval system](https://freenetproject.org/assets/papers/ddisrs.pdf)
-* [Protecting Free Expression Online with Freenet](https://freenetproject.org/assets/papers/ddisrs.pdf)
-* [Bitmessage: A Peer‐to‐Peer Message Authentication and Delivery System](https://archive.org/details/BitmessageWhitepaper/)
-* [MuON: Epidemic based Mutual Anonymity](https://web.archive.org/web/20060901153544/http://www.csl.mtu.edu/cs6461/www/Reading/MuON_ICNP2005.pdf)

docs/dev/setup.md

@@ -1,72 +0,0 @@
-<h1 align="center">Onionr Developer Guide</h1>
-
-This page assumes that Onionr is already installed and normal user requirements are setup.
-
-The Onionr development environment is simple. All one really needs is a supported Python version (currently 3.7-3.8 as of writing).
-
-There are additional requirements specified in requirements-dev.txt
-
-**Developers agree to the [CoC](../../CODE_OF_CONDUCT.md) and to contribute new code under GPLv3 or later**. Developers should stick to PEP8 in most cases, and write unittests or integration tests where possible.
-
-## Developer Scripts
-
-run-onionr-node.py can be used to start a node with specific parameters
-
-Intended to be used from VSCode (but could work otherwise), there are scripts in scripts/ named enable/disable-dev-config.py.
-These make modifications to the default config for the purpose of making testing Onionr nodes easier.
-Be sure to disable it again before pushing work.
-
-There are also scripts to generate new tests.
-
-*When adjusting PoW, it will make your node not compatible with the existing network*
-
-Generally, one should disable bootstrap list usage when testing non trivial changes. This is a config option: general.use_bootstrap_list. and can be configured through enable-dev-config.py and run-onionr-node.py
-
-
-# Current state of Onionr [2021-01-14]
-
-Onionr in it's current form is functional, albeit buggy.
-
-
-## Current major components
-
-Onionr runs via two main HTTP gevent servers serving Flask apps.
-
-Dir: apiservers
-
-* 1 Parent app hosts all public API endpoints for the Tor transport.
-* 1 Parent app hosts all UI-related files and endpoints. Some commands and internal modules interact with this API as well
-* The HTTP servers have strict anti-dns-rebinding and CSRF countermeasures, so there is a script to craft requests to the UI-related API in scripts/
-* Block storage is currently handled via metadata in sqlite (mostly defunct now), and block data storage in a different database. This is in blocks/ in running Onionr daemon data directory
-* cryptography is currently handled in onionrcrypto/ except for ephemeral messages which are handled by onionr
-* Transport clients run from looping threads mostly created in communicator/__init__.py, this includes block lookups and uploading on the Tor transport
-
-## Road map
-
-There are several big ways Onionr will be improved in the next major version:
-
-* Migration to the [new modular block system](https://git.voidnet.tech/kev/onionrblocks)
-    * Probability proof of work -> verifiable delay function
-    * Friend system built on top of signing proofs (Private networks?)
-* Gossip transport improvements such as with neighbor improvements. See streamfill/ and [simple gossip](https://github.com/onion-sudo/simplegossip) for incomplete experiments
-
-* Finish removing "communicator"
-* I2P transports
-    * Gossip
-    * Torrents (patch for sha1?)
-* Modular transports
-    * Currently transports are just threads coupled together.
-    * It would be better if there was a generic way to tell any loaded transport what blocks are wanted and feed back received blocks to the database
-* Migrate to SafeDB for peers and blocks
-    * SafeDB wrapper that contacts http endpoint to store if it is running, otherwise directly open DB
-* Separate UI logic from daemon. Refactor code to
-* Improve cryptography
-    * Restore phrases or deterministic keys (generate key from password, be very careful)
-    * Change identities to be dual keys (ed25519+curve25519)
-    * Finish treasurechest
-        * Interact via [named pipes](https://en.wikipedia.org/wiki/Named_pipe)
-        * Ephemeral key management
-        * Encrypt/decrypt/sign/verify functions to keep key out of main memory
-        * PGP-like symmetric messages
-
-

docs/dev/specs/block-spec.md

@@ -1,63 +0,0 @@
-# Onionr Block Spec v2.0.0
-
-# Block Description
-
-Onionr Blocks are the primary means of sharing information in Onionr. Blocks are identified by a single hash value of their entire contents, using SHA3_256.
-
-Blocks contain a JSON metadata section followed by a line break, with the main data section comprising the rest.
-
-In the future, the specification will likely be updated to use flags and MessagePack instead of JSON with english keys.
-
-# Encryption and Signatures
-
-Onionr blocks may be encrypted or signed. In the reference client, this is done with libsodium, for both asymmetric and symmetric encryption.
-
-Unlike many similar projects, blocks may completely be in plaintext, making Onionr suitable for sharing information publicly.
-
-# Metadata Section
-
-The metadata section has the following fields. If a block contains any other field, it must be considered invalid. All metadata fields are technically optional, but many are useful and essentially necessary for most use cases.
-
-## meta
-
-Max byte size (when in escaped json string format): 1000
-
-Meta is a string field which can contain arbitrary sub fields. It is intended for applications and plugins to use it for arbitrary metadata information. If the data section is encrypted or signed, the meta section also is.
-
-Common meta fields, such as 'type' are used by the reference Onionr client to describe the type of a block.
-
-## sig (optional)
-
-Max byte size: 200
-
-Sig is a field for storing public key signatures of the block, typically ed25519. In the reference client, this field is a base64 encoded signature of the meta field combined with the block data. (**Therefore, information outside of the meta and data fields cannot be trusted to be placed there by the signer, although it can still be assured that the particular block has not been modified.**)
-
-Note: the max field size is larger than a EdDSA signature (which is what is typically used) in order to allow other primitives for signing in alternative implementations or future versions.
-
-## signer (optional, necessary if sig is present)
-
-Max byte size: 200
-
-Signer is a field for specifying the public key which signed the block. In the reference client this is a base64 encoded ed25519 public key.
-
-## time (mandatory)
-
-Max byte size: 10
-
-Time is an integer field for specifying the time of which a block was created. The trustworthiness of this field is based on one's trust of the block creator, however blocks with a time field set in the future at the point of block receipt (past a reasonable clock skew) are thrown out by the reference client.
-
-## expire (optional)
-
-Max byte size: 10
-
-Expire is an integer field for specifying the time of which the block creator has indicated that the block should be deleted. The purpose of this is for voluntarily freeing the burden of unwanted blocks on the Onionr network, rather than security/privacy (since blocks could be trivially kept past expiration). Regardless, the reference client deletes blocks after a preset time if the expire field is either not set or longer than the preset time.
-
-## pow (effectively mandatory)
-
-Max byte size: 1000
-
-Pow is a field for placing the nonce found to make a block meet a target proof of work. In theory, a block could meet a target without a random token in this field.
-
-## encryptType (optional)
-
-encryptType is a field to specify the mode of encryption for a block. The values supported by Onionr are 'asym' and 'sym'.

docs/dev/specs/forward-secrecy.md

@@ -1,5 +0,0 @@
-# Onionr Forward Secrecy Spec v0.0.0
-
-# Introduction
-
-Due to the natural trade-offs of implementing [forward secrecy](https://en.wikipedia.org/wiki/Forward_secrecy) in a distributed, decentralized system, Onionr has optional ephemeral keys.

docs/docker.md

@@ -1,28 +0,0 @@
-# Running Onionr in Docker
-
-A Dockerfile is included in the root directory of Onionr.
-
-In Docker version 20.10 (and probably others), there is a strange bug where Onionr must be run with -it or stdout will be garbled and it may hang.
-
-## Clone and build the image
-
-`$ git clone https://git.voidnet.tech/kev/onionr/`
-`$ cd onionr`
-`$ sudo docker build -t onionr .`
-
-
-## Run Onionr
-
-`$ sudo docker run -it -p 8080:8080 onionr`
-
-Onionr will be accessible over any network interface by default, so make sure to either change the entry point bind-address argument or set a firewall rule.
-
-That said, Onionr does protect it's interface by default with a web token, which will be shown in stdout.
-
-**However, anyone who can access the port may be able to see what Onionr sites you have saved and potentially deanonymize your node**
-
-## View the UI
-
-Visit the address and port for the machine Onionr is running on, for example: http://192.168.1.5:8080/#<long-token-taken-from-stdout>
-
-If you want a secure connection to the interface, either use a proxy such as nginx or caddy, or use [SSH tunneling](./vps-cloud-guide.md).

docs/faq.md

@@ -1,24 +0,0 @@
-<h1 align="center">Onionr FAQ</h1>
-
-(Most of the FAQ answers apply to planned features and do not reflect in the demo network)
-
-* How does Onionr route messages?
-
-Onionr creates a message with an anti-spam proof (usually). It is then forwarded between nodes using binomial tree broadcast.
-
-The network is structured based on the particular transport being used, for Tor, .onion addresses are used to determine closeness between nodes.
-
-Churn due to connectivity changes or issues, membership changes or malicious activity means nodes need to sometimes look far away for missed messages.
-
-To help with scale, messages can be intelligently deleted or forwarded, for example a mail message can be marked by its recipient at a given point for deletion, or forum posts can be set to expire after a short period or after lack of activity.
-
-
-* Why do you use Python instead of [language]?
-
-I'm very comfortable in Python, and I believe it is a maintainable language if you use it correctly with modern features, also it has many quality libraries useful to the project, like pyncal and stem. Most places in the project are IO bound.
-
-
-* What do you think of bad actors who might use Onionr?
-
-Users should be able to exclude viewing, processing, and hosting content they do not want to. To this end, I want to enable user-configurable filtering based on lists provided by friends, fuzzy hashing, and voting.
-

docs/usage/deterministic.md

@@ -1,7 +0,0 @@
-# Deterministic address
-
-Deterministic addresses are simply Onionr user IDs that are generated from a passphrase rather than general entropy.
-
-To generate an Onionr deterministic, create a random passphrase of at least 25 characters of length. **It is very important that your passphrase is highly random. 9+ words is recommended for your passphrase**
-
-`$ onionr add-id true`

docs/usage/firststeps.md

@@ -1,7 +0,0 @@
-# Onionr First Steps
-
-After installing Onionr, there are several things to do:
-
-1. Setup a [deterministic address](deterministic.md) (optional)
-2. Start your node `/path/to/onionr.sh start`
-3. Copy your ID and share it with friends

docs/usage/install.md

@@ -1,18 +0,0 @@
-# Onionr Installation
-
-The following steps work broadly speaking for WSL, Mac, and Linux.
-
-1. Verify python3.7+ is installed: if not, see https://www.python.org/downloads/
-
-2. Verify Tor is installed (does not need to be running, binary can be put into system path or Onionr directory)
-
-3. [Optional but recommended]: setup virtual environment using [virtualenv](https://virtualenv.pypa.io/en/latest/), activate the virtual environment
-
-4. Clone Onionr: `$ git clone https://gitlab.com/beardog/onionr`
-
-5. Install the Python module dependencies: `$ pip3 install --require-hashes -r requirements.txt`
-
-
-Note: if an alternative python install is needed, use virtualenv or run Onionr commands with:
-
-`$ /path/to/python /path/to/onionr/onionr/__init__.py`

docs/usage/mail.md

@@ -1,16 +0,0 @@
-To use OnionrMail, open the web interface:
-
-`$ onionr.sh openweb`
-
-Navigate to the mail interface from the navbar.
-
-## Receiving mail
-
-Copy your ID from the textbox below the navbar. Publish the ID so that someone (or general people) know your ID to send messages to it. Note that associating the ID with your real ID publicly removes anonymity.
-
-## Sending mail
-
-1. optionally add the recipient as a friend via the friends page.
-2. click compose in the mail menu
-3. either select the recipient from the friend list or paste their ID in manually into the recipient box
-4. send away!

docs/usage/pages.md

@@ -1,59 +0,0 @@
-Onionr sites come in two forms:
-
-* Single-page sites, identified by the hash of a single page contained within a single Onionr block.
-
-* Multi-page sites, identified by a user ID. Contains directory archives of a full site.
-
-
-# Metadata Awareness
-
-Before creating an Onionr site, one should be cautious of the metadata one could be leaking. For example, some HTML generators may insert author meta tags. Onionr does not filter out any web page data.
-
-# No JavaScript, no third-party resources
-
-Currently, in order to protect Onionr users, JavaScript is disabled within Onionr sites. JS will remain present in the HTML file, but be non functional. Additionally, third party resources outside of Onionr cannot be loaded.
-
-
-# Creating multi page sites
-
-Multi page sites are the most useful, as they can contain an arbitrary amount of static files.
-
-To create a single page site, create a directory for your site and write standard HTML file(s) within them. CSS, images and other files can be placed in the directory as well. The home page should be name index.html and in the parent level directory.
-
-Then, create a strong passphrase for the site. If the site will be updated, be sure to write it down or remember it. A strong passphrase can be generated by running:
-
-`$ scripts/passphrase-generator.py`
-
-Sample output: lovesick blubberer haemoglobin... and so on.
-
-## Generating or updating the site:
-
-`$ ./onionr.sh addsite`
-
-All files in the current working directory will be added to the site.
-
-The command will prompt for a passphrase.
-
-After the site is generated, a user ID that identifies the site will be outputted.
-
-# Creating single page sites
-
-Single page sites are incredibly straight forward.
-
-Single page sites cannot be modified or updated, but are somewhat more secure due to having lower complexity.
-
-To create a single page site, write a standard HTML file. Inline or data-uri CSS can be included, as well as data-uri images. Data-URI generators can be found online.
-
-After creating the HTML file, run this command:
-
-`$ ./onionr.sh addhtml filename.html`
-
-![single page screenshot](single-page.png)
-
-# Viewing sites
-
-To view a site, open the Onionr web interface and paste the site hash or ID into the site opener box that looks like this:
-
-![site opener box screenshot](site-opener.png)
-
-Then, press open.

docs/usage/safety.md

@@ -1,6 +0,0 @@
-Onionr and all privacy-oriented software are not silver bullets.
-
-For example, if one gives away too much about themselves on an Onionr forum, it doesn't matter how good Onionr's privacy is.
-
-
-TODO: expand this page

docs/vps-cloud-guide.md

@@ -1,18 +0,0 @@
-# Cloud/Server Hosting Onionr
-
-Cloud hosting is not the recommended way to run Onionr, as it gives the cloud provider control over your data.
-
-That said, it is quite useful and by running a 24/7 Onionr node you contribute to the network's health.
-
-![https://cock.li/img/cockbox.png](https://cockbox.org/?r=12)
-
-[Cockbox](https://cockbox.org/?r=12) is the recommended provider, as they do not demand personal information and accept Monero. This is an affiliate link, which gives us a commission at no expense or complication to you.
-
-
-1. [Install Onionr like normal](./basic-onionr-user-guide.pdf) or [from Docker](./docker.md)
-2. Run with specific host and port to bind to: `$ ./run-onionr-node.py --bind-address 0.0.0.0 --port 8080`
-3. Get the web security token: `$ ./onionr.sh get-web` (the long string at the end of the URL)
-4. [Configure Firefox for SSH tunneling](https://web.archive.org/web/20210123034529/https://gist.github.com/brentjanderson/6ed800376e53746d2d28ba7b6bdcdc12). `Set network.proxy.allow_hijacking_localhost` to true in about:config
-
-
-Disable random host binding in config (general.security.random_bind_ip = false) if you have Onionr auto start. You may also want to disable deniable block inserts.

docs/whitepaper.md

@@ -1,18 +1,15 @@
 <p align="center">
-   <h1 align="center">Onionr</h1>
+ <img src="onionr-logo.png" alt="<h1>Onionr</h1>" width=200>
 </p>
-
 <p align="center">Anonymous, Decentralized, Distributed Network</p>
 
-June 2020
-
 # Introduction
 
 We believe that the ability to communicate freely with others is crucial for maintaining societal and personal liberty. The internet has provided humanity with the ability to spread information globally, but there are many persons and organizations who try to stifle the flow of information, sometimes with success.
 
-Internet censorship comes in many forms, state censorship, threats of violence, network exploitation (e.g. denial of service attacks) and others.
+Internet censorship comes in many forms, state censorship, corporate consolidation of media, threats of violence, network exploitation (e.g. denial of service attacks) and other threats.
 
-We hold that in order to protect individual privacy, users must have the ability to communicate anonymously and with decentralization.
+We hold that in order to protect individual privacy, users must have the ability to communicate anonymously and with decentralization. 
 
 We believe that in order to prevent censorship and loss of information, these measures must be in place:
 
@@ -42,11 +39,11 @@ When designing Onionr we had these main goals in mind:
 
 At its core, Onionr is merely a description for storing data in self-verifying packages ("blocks"). These blocks can be encrypted to a user (or for one's self), encrypted symmetrically, or not at all. Blocks can be signed by their creator, but regardless, they are self-verifying due to being identified by a sha3-256 hash value; once a block is created, it cannot be modified.
 
-Onionr exchanges a list of blocks between all nodes. By default, all nodes download and share all other blocks, however, this is configurable. Blocks do not rely on any particular order of receipt or transport mechanism.
+Onionr exchanges a list of blocks between all nodes. By default, all nodes download and share all other blocks, however this is configurable. Blocks do not rely on any particular order of receipt or transport mechanism.
 
 ## User IDs
 
-User IDs are simply Ed25519 public keys. They are represented in Base32 format or encoded using the [PGP Word List](https://en.wikipedia.org/wiki/PGP_word_list).
+User IDs are simply Ed25519 public keys. They are represented in Base32 format, or encoded using the [PGP Word List](https://en.wikipedia.org/wiki/PGP_word_list).
 
 Public keys can be generated deterministically with a password using a key derivation function (Argon2id). This password can be shared between many users in order to share data anonymously among a group, using only 1 password. This is useful in some cases, but is risky, as if one user causes the key to be compromised and does not notify the group or revoke the key, there is no way to know.
 
@@ -56,21 +53,19 @@ Although Onionr is transport agnostic, the only supported transports in the refe
 
 ### Node Profiling
 
-To mitigate maliciously slow or unreliable nodes, Onionr builds a profile on nodes it connects to. Nodes are assigned a score, which raises based on the number of successful block transfers, speed, and reliability of a node, and reduces the score based on how unreliable a node is. If a node is unreachable for over 24 hours after contact, it is forgotten. Onionr can also prioritize connections to 'friend' nodes.
+To mitigate maliciously slow or unreliable nodes, Onionr builds a profile on nodes it connects to. Nodes are assigned a score, which raises based on the amount of successful block transfers, speed, and reliability of a node, and reduces the score based on how unreliable a node is. If a node is unreachable for over 24 hours after contact, it is forgotten. Onionr can also prioritize connection to 'friend' nodes.
 
 ## Block Format
 
-Onionr blocks are very simple. They are structured in two main parts: a metadata section and a data section, with a line feed delimiting where metadata ends and data begins.
+Onionr blocks are very simple. They are structured in two main parts: a metadata section and a data section, with a line feed delimiting where metadata ends and data begins. 
 
 Metadata defines what kind of data is in a block, signature data, encryption settings, and other arbitrary information.
 
 Optionally, a random token can be inserted into the metadata for use in Proof of Work.
 
-The proof of work function should be a Verifiable Delay Function (VDF). We have chosen MiMC for it's simplicity.
-
 ### Block Encryption
 
-For encryption, Onionr uses ephemeral Curve25519 keys for key exchange and XSalsa20-Poly1305 as a symmetric cipher or optionally using only XSalsa20-Poly1305 with a pre-shared key.
+For encryption, Onionr uses ephemeral Curve25519 keys for key exchange and XSalsa20-Poly1305 as a symmetric cipher, or optionally using only XSalsa20-Poly1305 with a pre-shared key.
 
 Regardless of encryption, blocks can be signed internally using Ed25519.
 
@@ -88,19 +83,21 @@ Blocks are stored indefinitely until the allocated space is filled, at which poi
 
 ## Block Timestamping
 
-Onionr blocks are by default not accepted if their timestamp is set too far in the past, or is in the future.
+Onionr can provide evidence of when a block was inserted by requesting other users to sign a hash of the current time with the block data hash: sha3_256(time + sha3_256(block data)).
+
+This can be done either by the creator of the block prior to generation, or by any node after insertion.
 
 In addition, randomness beacons such as the one operated by [NIST](https://beacon.nist.gov/home), [Chile](https://beacon.clcert.cl/), or the hash of the latest blocks in a cryptocurrency network could be used to affirm that a block was at least not *created* before a given time.
 
 # Direct Connections
 
-We propose a method of using Onionr's block sync system to enable direct connections between peers by having one peer request to connect to another using the peer's public key. Since the request is within a standard block, proof of work must be used to request a connection. If the requested peer is available and wishes to accept the connection, Onionr will generate a temporary .onion address for the other peer to connect to. Alternatively, a reverse connection may be formed, which is faster to establish but requires a message brokering system instead of a standard socket.
+We propose a method of using Onionr's block sync system to enable direct connections between peers by having one peer request to connect to another using the peer's public key. Since the request is within a standard block, proof of work must be used to request connection. If the requested peer is available and wishes to accept the connection, Onionr will generate a temporary .onion address for the other peer to connect to. Alternatively, a reverse connection may be formed, which is faster to establish but requires a message brokering system instead of a standard socket.
 
 The benefits of such a system are increased privacy, and the ability to anonymously communicate from multiple devices at once. In a traditional onion service, one's online status can be monitored and more easily correlated.
 
 # Threat Model
 
-The goal of Onionr is to provide a method of distributing information in a manner in which the difficulty of discovering the identity of those sending and receiving the information is greatly increased. In this section we detail what information we want to protect and who we're protecting it from.
+The goal of Onionr is to provide a method of distributing information in a manner in which the difficulty of discovering the identity of those sending and receiving the information is greatly increased. In this section we detail what information we want to protect and who we're protecting it from. 
 
 In this threat model, "protected" means available in plaintext only to those which it was intended, and regardless non-malleable
 
@@ -122,9 +119,7 @@ We seek to protect the following information:
 * Physical location/IP address of nodes on the network
 * All block data from tampering
 
-### Unprotected Data
-
-Onionr does not protect the following:
+### Data we cannot or do not protect
 
 * Data specifically inserted as plaintext is available to the public
 * The public key of signed plaintext blocks
@@ -134,10 +129,16 @@ Onionr does not protect the following:
 
 ## Assumptions
 
-We assume that Tor onion services (v3) and I2P services cannot be trivially deanonymized, and that the underlying cryptographic primitives we employ cannot be broken in any manner faster than brute force unless a quantum computer is used.
+We assume that Tor onion services (v3) and I2P services cannot be trivially deanonymized, and that the underlying cryptographic primitives we employ cannot be broken in any manner faster than brute force unless a quantum computer is used. 
 
 Once quantum safe algorithms are more mature and have decent high level libraries, they will be deployed.
 
+# Comparisons to other P2P software
+
+Since Onionr is far from the first to implement many of these ideas (on their own), this section compares Onionr to other networks, using points we consider to be the most important.
+
+![network comparison image](network-comparison.png)
+
 # Conclusion
 
-If successful, Onionr will be a complete decentralized platform for anonymous computing, complete with limited metadata exposure, both node and user anonymity, and spam prevention
+If successful, Onionr will be a complete decentralized platform for anonymous computing, complete with limited metadata exposure, both node and user anonymity, and spam prevention

git-update.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-echo "This will git pull origin master, then update dependencies and default plugins."
-echo "Enter to continue, ctrl-c to stop."
-read
-
-git pull origin master
-pip3 install -r --require-hashes requirements.txt
-./onionr.sh reset-plugins

install/CentOS-install

@@ -1,35 +0,0 @@
-yum install epel-release
-yum clean all
-yum update -y
-
-yum group install "Development Tools"
-yum install centos-release-scl
-yum install wget zlib zlib-devel openssl-devel bzip2-devel libffi-devel python-gevent -y
-yum install rh-python38 rh-python38-python-devel
-scl enable rh-python38 bash
-yum update -y
-
-pip3 install --require-hashes -r requirements.txt
-chown nobody. ../
-su -l nobody -s /bin/bash
-scl enable rh-python38 bash
-./onionr.sh start
-
-Like a black hole, NSA pulls in every signal that comes near, but no electron is ever allowed to escape│
-│ -James Bamford                                                                                        │
-└───────────────────────────────────────────────────────────────────────────────────────────────────────┘
-[+] Tor is starting...
-[+] Jan 11 18:35:55.000 [notice] Bootstrapped 0%: Starting
-[+] Jan 11 18:35:56.000 [notice] Bootstrapped 10%: Finishing handshake with directory server
-[+] Jan 11 18:35:56.000 [notice] Bootstrapped 80%: Connecting to the Tor network
-[+] Jan 11 18:35:56.000 [notice] Bootstrapped 90%: Establishing a Tor circuit
-[+] Finished starting Tor.
-[+] Onionr v8.0.1 (x86_64) (API v2)
-[+] Private P2P Communication - GPLv3 - https://Onionr.net
-[+] CPython 3.8.6 on Linux-3.10.0-1062.4.1.el7.x86_64-x86_64-with-glibc2.2.5 3.10.0-1062.4.1.el7.x86_64
-[+] Onionr data dir: /root/.local/share/onionr/
-[+] Onionr daemon is running under 5186
-[+] Not binding to LAN due to no private network configured.
-[+] First run detected. Run openhome to get setup.
-
-./onionr.sh openhome

install/README.md

@@ -1,5 +0,0 @@
-# Service/System Installation Notes
-
-Currently, the most reliable and well tested way to use Onionr is straight from the master branch of the repository, or from the released bundles.
-
-System services and system-wide installation is not supported well yet.

install/build.sh

@@ -1,5 +1,4 @@
 #!/bin/sh
 
-sudo pip3 install -r requirements.txt
 make plugins-reset
 find . -name '__pycache__' -type d | xargs rm -rf

install/install_arch.sh

@@ -5,7 +5,7 @@ OUTPUT_DIR='/usr/share/onionr'
 DATA_DIR='/etc/onionr'
 LOG_DIR='/var/log/onionr'
 
-BRANCH='master'
+BRANCH='easy-releases'
 
 # setup error handlers
 
@@ -80,7 +80,7 @@ systemctl start onionr
 
 # pretty header thing
 
-"$EXECUTABLE" details
+"$EXECUTABLE" --header 'Onionr successfully installed.'
 
 # and we're good!
 

install/install_debian.sh

@@ -5,7 +5,7 @@ OUTPUT_DIR='/usr/share/onionr'
 DATA_DIR='/etc/onionr'
 LOG_DIR='/var/log/onionr'
 
-BRANCH='master'
+BRANCH='easy-releases'
 
 # setup error handlers
 

install/onionr

@@ -1,16 +1,23 @@
 #!/bin/sh
+
 set -e
 
-if [[ $EUID -eq 0 ]]; then
-    export ONIONR_HOME=${ONIONR_HOME:-/var/lib/onionr}
-    export LOG_DIR=${LOG_DIR:-/var/log/onionr}
+export OUTPUT_DIR=${OUTPUT_DIR:=/usr/share/onionr}
+
+if [ -n "$HOME" ]; then
+    export XDG_DATA_HOME=${XDG_DATA_HOME:=$HOME/.local/share/onionr}
+
+    export ONIONR_HOME=${ONIONR_HOME:=$XDG_DATA_HOME}
+    export LOG_DIR=${LOG_DIR:=$XDG_DATA_HOME/logs}
 else
-    export ONIONR_HOME=${ONIONR_HOME:-${XDG_DATA_HOME:-$HOME/.local/share}/onionr}
-    export LOG_DIR=${LOG_DIR:-$ONIONR_HOME/logs}
+    export ONIONR_HOME=${ONIONR_HOME:=/etc/onionr}
+    export LOG_DIR=${LOG_DIR:=/var/log/onionr}
 fi
 
 mkdir -p "$ONIONR_HOME" "$LOG_DIR"
 
-chmod 0700 "$ONIONR_HOME" "$LOG_DIR"
+chmod -R 700 "$ONIONR_HOME" "$LOG_DIR"
+chown -R $USER:$USER "$ONIONR_HOME" "$LOG_DIR"
 
-exec ${ONIONR_BASEDIR:-/usr/share/onionr}/onionr.sh "$@"
+cd "$OUTPUT_DIR/onionr"
+exec python3.7 onionr.py "$@"

install/onionr.service

@@ -1,20 +1,22 @@
 [Unit]
 Description=Onionr Daemon
 Documentation=https://onionr.net/docs/
-After=network-online.target
-Requires=network-online.target
+After=network.target tor.service
+Requires=network.target tor.service systemd-networkd-wait-online.service
 
 [Service]
-Environment="ONIONR_HOME=/var/lib/onionr"
-Environment="LOG_DIR=/var/log/onionr"
+Environment="DATA_DIR=/usr/share/onionr"
+Environment="LOG_DIR=/var/log/onionr/"
 
-ExecStart=/usr/bin/onionr start
-ExecStop=/usr/bin/onionr stop
+ExecStart=/usr/bin/onionr --start
+ExecStop=/usr/bin/onionr --stop
 
 KillMode=mixed
 KillSignal=SIGQUIT
-TimeoutStopSec=30s
+TimeoutStopSec=5s
+Type=simple
 Restart=on-abnormal
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=tor.service
+

make-release.sh

@@ -1,15 +0,0 @@
-#!/usr/bin/bash
-rm -rf dist
-mkdir dist
-mkdir dist/onionr/
-cp -t dist/onionr/ -r docs static-data install src onionr.sh start-daemon.sh setprofile.sh requirements.txt requirements-notifications.txt
-cp *.md dist/onionr/
-PIP_USER=false
-export PIP_USER
-cd dist
-tar -czvf onionr-no-deps.tar.gz onionr
-cd ..
-pip3 install --require-hashes -r requirements.txt --target=dist/onionr/src/
-pip3 install --require-hashes -r requirements-notifications.txt --target=dist/onionr/src/
-cd dist
-tar -czvf onionr.tar.gz onionr

onionr.sh

@@ -1,9 +1,4 @@
 #!/bin/sh
-ORIG_ONIONR_RUN_DIR=`pwd`
-export ORIG_ONIONR_RUN_DIR
-export PYTHONDONTWRITEBYTECODE=1
-export PYTHONUNBUFFERED=1
-export PYTHONOPTIMIZE="true"
 cd "$(dirname "$0")"
-cd src
-./__init__.py "$@"
+cd onionr/
+./onionr.py "$@"

onionr.yml

@@ -1,6 +1,4 @@
-# lazypkg config
-
-name: onionr-git
+name: onionr
 version: 0.1
 release: 1
 summary: anonymous P2P communication platform
@@ -9,15 +7,14 @@ description: Onionr is a decentralized, peer-to-peer communication network, desi
 license: GPL
 website: https://onionr.net/
 contact: contact@onionr.net
-maintainer: Aaron Esau
-maintainer-contact: aur@aaronesau.com
+maintainer: Kevin Froman
 
 relationships:
 - conflicts: onionr2
 
 sources:
 - git: https://gitlab.com/beardog/onionr.git
-  branch: master
+  branch: easy-releases
 
 dependencies:
 - deb: git

onionr/api.py

@@ -0,0 +1,602 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This file handles all incoming http requests to the client, using Flask
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+from gevent.pywsgi import WSGIServer, WSGIHandler
+from gevent import Timeout
+import flask, cgi, uuid
+from flask import request, Response, abort, send_from_directory
+import sys, random, threading, hmac, base64, time, os, json, socket
+import core
+from onionrblockapi import Block
+import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config
+import httpapi
+from httpapi import friendsapi, simplecache
+import onionr
+
+class FDSafeHandler(WSGIHandler):
+    '''Our WSGI handler. Doesn't do much non-default except timeouts'''
+    def handle(self):
+       timeout = Timeout(60, exception=Exception)
+       timeout.start()
+       try:
+           WSGIHandler.handle(self)
+       except Timeout as ex:
+           raise
+
+def setBindIP(filePath):
+    '''Set a random localhost IP to a specified file (intended for private or public API localhost IPs)'''
+    if config.get('general.random_bind_ip', True):
+        hostOctets = [str(127), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF))]
+        data = '.'.join(hostOctets)
+        # Try to bind IP. Some platforms like Mac block non normal 127.x.x.x
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        try:
+            s.bind((data, 0))
+        except OSError:
+            # if mac/non-bindable, show warning and default to 127.0.0.1
+            logger.warn('Your platform appears to not support random local host addresses 127.x.x.x. Falling back to 127.0.0.1.')
+            data = '127.0.0.1'
+        s.close()
+    else:
+        data = '127.0.0.1'
+    with open(filePath, 'w') as bindFile:
+        bindFile.write(data)
+    return data
+
+class PublicAPI:
+    '''
+        The new client api server, isolated from the public api
+    '''
+    def __init__(self, clientAPI):
+        assert isinstance(clientAPI, API)
+        app = flask.Flask('PublicAPI')
+        self.i2pEnabled = config.get('i2p.host', False)
+        self.hideBlocks = [] # Blocks to be denied sharing
+        self.host = setBindIP(clientAPI._core.publicApiHostFile)
+        self.torAdder = clientAPI._core.hsAddress
+        self.i2pAdder = clientAPI._core.i2pAddress
+        self.bindPort = config.get('client.public.port')
+        self.lastRequest = 0
+        logger.info('Running public api on %s:%s' % (self.host, self.bindPort))
+
+        @app.before_request
+        def validateRequest():
+            '''Validate request has the correct hostname'''
+            # If high security level, deny requests to public (HS should be disabled anyway for Tor, but might not be for I2P)
+            if config.get('general.security_level', default=0) > 0:
+                abort(403)
+            if type(self.torAdder) is None and type(self.i2pAdder) is None:
+                # abort if our hs addresses are not known
+                abort(403)
+            if request.host not in (self.i2pAdder, self.torAdder):
+                # Disallow connection if wrong HTTP hostname, in order to prevent DNS rebinding attacks
+                abort(403)
+
+        @app.after_request
+        def sendHeaders(resp):
+            '''Send api, access control headers'''
+            resp.headers['Date'] = 'Thu, 1 Jan 1970 00:00:00 GMT' # Clock info is probably useful to attackers. Set to unix epoch, since we can't fully remove the header.
+            # CSP to prevent XSS. Mainly for client side attacks (if hostname protection could somehow be bypassed)
+            resp.headers["Content-Security-Policy"] =  "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'"
+            # Prevent click jacking
+            resp.headers['X-Frame-Options'] = 'deny'
+            # No sniff is possibly not needed
+            resp.headers['X-Content-Type-Options'] = "nosniff"
+            # Network API version
+            resp.headers['X-API'] = onionr.API_VERSION
+            # Close connections to limit FD use
+            resp.headers['Connection'] = "close"
+            self.lastRequest = clientAPI._core._utils.getRoundedEpoch(roundS=5)
+            return resp
+
+        @app.route('/')
+        def banner():
+            # Display a bit of information to people who visit a node address in their browser
+            try:
+                with open('static-data/index.html', 'r') as html:
+                    resp = Response(html.read(), mimetype='text/html')
+            except FileNotFoundError:
+                resp = Response("")
+            return resp
+
+        @app.route('/getblocklist')
+        def getBlockList():
+            # Provide a list of our blocks, with a date offset
+            dateAdjust = request.args.get('date')
+            bList = clientAPI._core.getBlockList(dateRec=dateAdjust)
+            for b in self.hideBlocks:
+                if b in bList:
+                    # Don't share blocks we created if they haven't been *uploaded* yet, makes it harder to find who created a block
+                    bList.remove(b)
+            return Response('\n'.join(bList))
+
+        @app.route('/getdata/<name>')
+        def getBlockData(name):
+            # Share data for a block if we have it
+            resp = ''
+            data = name
+            if clientAPI._utils.validateHash(data):
+                if data not in self.hideBlocks:
+                    if data in clientAPI._core.getBlockList():
+                        block = clientAPI.getBlockData(data, raw=True)
+                        try:
+                            block = block.encode()
+                        except AttributeError:
+                            abort(404)
+                        block = clientAPI._core._utils.strToBytes(block)
+                        resp = block
+                        #resp = base64.b64encode(block).decode()
+            if len(resp) == 0:
+                abort(404)
+                resp = ""
+            return Response(resp, mimetype='application/octet-stream')
+
+        @app.route('/www/<path:path>')
+        def wwwPublic(path):
+            # A way to share files directly over your .onion
+            if not config.get("www.public.run", True):
+                abort(403)
+            return send_from_directory(config.get('www.public.path', 'static-data/www/public/'), path)
+
+        @app.route('/ping')
+        def ping():
+            # Endpoint to test if nodes are up
+            return Response("pong!")
+
+        @app.route('/pex')
+        def peerExchange():
+            response = ','.join(clientAPI._core.listAdders(recent=3600))
+            if len(response) == 0:
+                response = ''
+            return Response(response)
+        
+        @app.route('/announce', methods=['post'])
+        def acceptAnnounce():
+            resp = 'failure'
+            powHash = ''
+            randomData = ''
+            newNode = ''
+            ourAdder = clientAPI._core.hsAddress.encode()
+            try:
+                newNode = request.form['node'].encode()
+            except KeyError:
+                logger.warn('No node specified for upload')
+                pass
+            else:
+                try:
+                    randomData = request.form['random']
+                    randomData = base64.b64decode(randomData)
+                except KeyError:
+                    logger.warn('No random data specified for upload')
+                else:
+                    nodes = newNode + clientAPI._core.hsAddress.encode()
+                    nodes = clientAPI._core._crypto.blake2bHash(nodes)
+                    powHash = clientAPI._core._crypto.blake2bHash(randomData + nodes)
+                    try:
+                        powHash = powHash.decode()
+                    except AttributeError:
+                        pass
+                    if powHash.startswith('0000'):
+                        newNode = clientAPI._core._utils.bytesToStr(newNode)
+                        if clientAPI._core._utils.validateID(newNode) and not newNode in clientAPI._core.onionrInst.communicatorInst.newPeers:
+                            clientAPI._core.onionrInst.communicatorInst.newPeers.append(newNode)
+                            resp = 'Success'
+                    else:
+                        logger.warn(newNode.decode() + ' failed to meet POW: ' + powHash)
+            resp = Response(resp)
+            return resp
+
+        @app.route('/upload', methods=['post'])
+        def upload():
+            '''Accept file uploads. In the future this will be done more often than on creation 
+            to speed up block sync
+            '''
+            resp = 'failure'
+            try:
+                data = request.form['block']
+            except KeyError:
+                logger.warn('No block specified for upload')
+                pass
+            else:
+                if sys.getsizeof(data) < 100000000:
+                    try:
+                        if blockimporter.importBlockFromData(data, clientAPI._core):
+                            resp = 'success'
+                        else:
+                            logger.warn('Error encountered importing uploaded block')
+                    except onionrexceptions.BlacklistedBlock:
+                        logger.debug('uploaded block is blacklisted')
+                        pass
+            if resp == 'failure':
+                abort(400)
+            resp = Response(resp)
+            return resp
+
+        # Set instances, then startup our public api server
+        clientAPI.setPublicAPIInstance(self)
+        while self.torAdder == '':
+            clientAPI._core.refreshFirstStartVars()
+            self.torAdder = clientAPI._core.hsAddress
+            time.sleep(0.1)
+        self.httpServer = WSGIServer((self.host, self.bindPort), app, log=None, handler_class=FDSafeHandler)
+        self.httpServer.serve_forever()
+
+class API:
+    '''
+        Client HTTP api
+    '''
+
+    callbacks = {'public' : {}, 'private' : {}}
+
+    def __init__(self, onionrInst, debug, API_VERSION):
+        '''
+            Initialize the api server, preping variables for later use
+
+            This initilization defines all of the API entry points and handlers for the endpoints and errors
+            This also saves the used host (random localhost IP address) to the data folder in host.txt
+        '''
+
+        self.debug = debug
+        self._core = onionrInst.onionrCore
+        self.startTime = self._core._utils.getEpoch()
+        self._crypto = onionrcrypto.OnionrCrypto(self._core)
+        self._utils = onionrutils.OnionrUtils(self._core)
+        app = flask.Flask(__name__)
+        bindPort = int(config.get('client.client.port', 59496))
+        self.bindPort = bindPort
+
+        # Be extremely mindful of this. These are endpoints available without a password
+        self.whitelistEndpoints = ('site', 'www', 'onionrhome', 'board', 'boardContent', 'sharedContent', 'mail', 'mailindex', 'friends', 'friendsindex')
+
+        self.clientToken = config.get('client.webpassword')
+        self.timeBypassToken = base64.b16encode(os.urandom(32)).decode()
+
+        self.publicAPI = None # gets set when the thread calls our setter... bad hack but kinda necessary with flask
+        #threading.Thread(target=PublicAPI, args=(self,)).start()
+        self.host = setBindIP(self._core.privateApiHostFile)
+        logger.info('Running api on %s:%s' % (self.host, self.bindPort))
+        self.httpServer = ''
+
+        self.pluginResponses = {} # Responses for plugin endpoints
+        self.queueResponse = {}
+        onionrInst.setClientAPIInst(self)
+        app.register_blueprint(friendsapi.friends)
+        app.register_blueprint(simplecache.simplecache)
+        httpapi.load_plugin_blueprints(app)
+
+        @app.before_request
+        def validateRequest():
+            '''Validate request has set password and is the correct hostname'''
+            # For the purpose of preventing DNS rebinding attacks
+            if request.host != '%s:%s' % (self.host, self.bindPort):
+                abort(403)
+            if request.endpoint in self.whitelistEndpoints:
+                return
+            try:
+                if not hmac.compare_digest(request.headers['token'], self.clientToken):
+                    if not hmac.compare_digest(request.form['token'], self.clientToken):
+                        abort(403)
+            except KeyError:
+                if not hmac.compare_digest(request.form['token'], self.clientToken):
+                    abort(403)
+
+        @app.after_request
+        def afterReq(resp):
+            # Security headers
+            if request.endpoint == 'site':
+                resp.headers['Content-Security-Policy'] = "default-src 'none'; style-src data: 'unsafe-inline'; img-src data:"
+            else:
+                resp.headers['Content-Security-Policy'] = "default-src 'none'; script-src 'self'; object-src 'none'; style-src 'self'; img-src 'self'; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'self'"
+            resp.headers['X-Frame-Options'] = 'deny'
+            resp.headers['X-Content-Type-Options'] = "nosniff"
+            resp.headers['Server'] = ''
+            resp.headers['Date'] = 'Thu, 1 Jan 1970 00:00:00 GMT' # Clock info is probably useful to attackers. Set to unix epoch.
+            resp.headers['Connection'] = "close"
+            return resp
+
+        @app.route('/board/', endpoint='board')
+        def loadBoard():
+            return send_from_directory('static-data/www/board/', "index.html")
+
+        @app.route('/mail/<path:path>', endpoint='mail')
+        def loadMail(path):
+            return send_from_directory('static-data/www/mail/', path)
+        @app.route('/mail/', endpoint='mailindex')
+        def loadMailIndex():
+            return send_from_directory('static-data/www/mail/', 'index.html')
+        
+        @app.route('/friends/<path:path>', endpoint='friends')
+        def loadContacts(path):
+            return send_from_directory('static-data/www/friends/', path)
+
+        @app.route('/friends/', endpoint='friendsindex')
+        def loadContacts():
+            return send_from_directory('static-data/www/friends/', 'index.html')
+
+        @app.route('/board/<path:path>', endpoint='boardContent')
+        def boardContent(path):
+            return send_from_directory('static-data/www/board/', path)
+        @app.route('/shared/<path:path>', endpoint='sharedContent')
+        def sharedContent(path):
+            return send_from_directory('static-data/www/shared/', path)
+
+        @app.route('/www/<path:path>', endpoint='www')
+        def wwwPublic(path):
+            if not config.get("www.private.run", True):
+                abort(403)
+            return send_from_directory(config.get('www.private.path', 'static-data/www/private/'), path)
+
+        @app.route('/queueResponseAdd/<name>', methods=['post'])
+        def queueResponseAdd(name):
+            # Responses from the daemon. TODO: change to direct var access instead of http endpoint
+            self.queueResponse[name] = request.form['data']
+            return Response('success')
+        
+        @app.route('/queueResponse/<name>')
+        def queueResponse(name):
+            # Fetch a daemon queue response
+            resp = 'failure'
+            try:
+                resp = self.queueResponse[name]
+            except KeyError:
+                pass
+            else:
+                del self.queueResponse[name]
+            return Response(resp)
+            
+        @app.route('/ping')
+        def ping():
+            # Used to check if client api is working
+            return Response("pong!")
+
+        @app.route('/', endpoint='onionrhome')
+        def hello():
+            # ui home
+            return send_from_directory('static-data/www/private/', 'index.html')
+        
+        @app.route('/getblocksbytype/<name>')
+        def getBlocksByType(name):
+            blocks = self._core.getBlocksByType(name)
+            return Response(','.join(blocks))
+        
+        @app.route('/getblockbody/<name>')
+        def getBlockBodyData(name):
+            resp = ''
+            if self._core._utils.validateHash(name):
+                try:
+                    resp = Block(name, decrypt=True).bcontent
+                    #resp =  cgi.escape(Block(name, decrypt=True).bcontent, quote=True)
+                except TypeError:
+                    pass
+            else:
+                abort(404)
+            return Response(resp)
+        
+        @app.route('/getblockdata/<name>')
+        def getData(name):
+            resp = ""
+            if self._core._utils.validateHash(name):
+                if name in self._core.getBlockList():
+                    try:
+                        resp = self.getBlockData(name, decrypt=True)
+                    except ValueError:
+                        pass
+                else:
+                    abort(404)
+            else:
+                abort(404)
+            return Response(resp)
+
+        @app.route('/getblockheader/<name>')
+        def getBlockHeader(name):
+            resp = self.getBlockData(name, decrypt=True, headerOnly=True)
+            return Response(resp)
+
+        @app.route('/lastconnect')
+        def lastConnect():
+            return Response(str(self.publicAPI.lastRequest))
+
+        @app.route('/site/<name>', endpoint='site')
+        def site(name):
+            bHash = name
+            resp = 'Not Found'
+            if self._core._utils.validateHash(bHash):
+                try:
+                    resp = Block(bHash).bcontent
+                except onionrexceptions.NoDataAvailable:
+                    abort(404)
+                except TypeError:
+                    pass
+                try:
+                    resp = base64.b64decode(resp)
+                except:
+                    pass
+            if resp == 'Not Found' or not resp:
+                abort(404)
+            return Response(resp)
+
+        @app.route('/waitforshare/<name>', methods=['post'])
+        def waitforshare(name):
+            '''Used to prevent the **public** api from sharing blocks we just created'''
+            assert name.isalnum()
+            if name in self.publicAPI.hideBlocks:
+                self.publicAPI.hideBlocks.remove(name)
+                return Response("removed")
+            else:
+                self.publicAPI.hideBlocks.append(name)
+                return Response("added")
+
+        @app.route('/shutdown')
+        def shutdown():
+            try:
+                self.publicAPI.httpServer.stop()
+                self.httpServer.stop()
+            except AttributeError:
+                pass
+            return Response("bye")
+
+        @app.route('/shutdownclean')
+        def shutdownClean():
+            # good for calling from other clients
+            self._core.daemonQueueAdd('shutdown')
+            return Response("bye")
+        
+        @app.route('/getstats')
+        def getStats():
+            # returns node stats
+            #return Response("disabled")
+            while True:
+                try:    
+                    return Response(self._core.serializer.getStats())
+                except AttributeError:
+                    pass
+        
+        @app.route('/getuptime')
+        def showUptime():
+            return Response(str(self.getUptime()))
+        
+        @app.route('/getActivePubkey')
+        def getActivePubkey():
+            return Response(self._core._crypto.pubKey)
+
+        @app.route('/getHumanReadable/<name>')
+        def getHumanReadable(name):
+            return Response(self._core._utils.getHumanReadableID(name))
+
+        @app.route('/insertblock', methods=['POST'])
+        def insertBlock():
+            encrypt = False
+            bData = request.get_json(force=True)
+            message = bData['message']
+            subject = 'temp'
+            encryptType = ''
+            sign = True
+            meta = {}
+            to = ''
+            try:
+                if bData['encrypt']:
+                    to = bData['to']
+                    encrypt = True
+                    encryptType = 'asym'
+            except KeyError:
+                pass
+            try:
+                if not bData['sign']:
+                    sign = False
+            except KeyError:
+                pass
+            try:
+                bType = bData['type']
+            except KeyError:
+                bType = 'bin'
+            try:
+                meta = json.loads(bData['meta'])
+            except KeyError:
+                pass
+            threading.Thread(target=self._core.insertBlock, args=(message,), kwargs={'header': bType, 'encryptType': encryptType, 'sign':sign, 'asymPeer': to, 'meta': meta}).start()
+            return Response('success')
+        
+        @app.route('/apipoints/<path:subpath>', methods=['POST', 'GET'])
+        def pluginEndpoints(subpath=''):
+            '''Send data to plugins'''
+            # TODO have a variable for the plugin to set data to that we can use for the response
+            pluginResponseCode = str(uuid.uuid4())
+            resp = 'success'
+            responseTimeout = 20
+            startTime = self._core._utils.getEpoch()
+            postData = {}
+            if request.method == 'POST':
+                postData = request.form['postData']
+            if len(subpath) > 1:
+                data = subpath.split('/')
+                if len(data) > 1:
+                    plName = data[0]
+                    events.event('pluginRequest', {'name': plName, 'path': subpath, 'pluginResponse': pluginResponseCode, 'postData': postData}, onionr=onionrInst)
+                    while True:
+                        try:
+                            resp = self.pluginResponses[pluginResponseCode]
+                        except KeyError:
+                            time.sleep(0.2)
+                            if self._core._utils.getEpoch() - startTime > responseTimeout:
+                                abort(504)
+                                break
+                        else:
+                            break
+            else:
+                abort(404)
+            return Response(resp)
+
+        self.httpServer = WSGIServer((self.host, bindPort), app, log=None, handler_class=FDSafeHandler)
+        self.httpServer.serve_forever()
+
+    def setPublicAPIInstance(self, inst):
+        assert isinstance(inst, PublicAPI)
+        self.publicAPI = inst
+
+    def validateToken(self, token):
+        '''
+            Validate that the client token matches the given token. Used to prevent CSRF and data exfiltration
+        '''
+        if len(self.clientToken) == 0:
+            logger.error("client password needs to be set")
+            return False
+        try:
+            if not hmac.compare_digest(self.clientToken, token):
+                return False
+            else:
+                return True
+        except TypeError:
+            return False
+
+    def getUptime(self):
+        while True:
+            try:
+                return self._utils.getEpoch - startTime
+            except AttributeError:
+                # Don't error on race condition with startup
+                pass
+    
+    def getBlockData(self, bHash, decrypt=False, raw=False, headerOnly=False):
+        assert self._core._utils.validateHash(bHash)
+        bl = Block(bHash, core=self._core)
+        if decrypt:
+            bl.decrypt()
+            if bl.isEncrypted and not bl.decrypted:
+                raise ValueError
+
+        if not raw:
+            if not headerOnly:
+                retData = {'meta':bl.bheader, 'metadata': bl.bmetadata, 'content': bl.bcontent}
+                for x in list(retData.keys()):
+                    try:
+                        retData[x] = retData[x].decode()
+                    except AttributeError:
+                        pass
+            else:
+                validSig = False
+                signer = self._core._utils.bytesToStr(bl.signer)
+                #print(signer, bl.isSigned(), self._core._utils.validatePubKey(signer), bl.isSigner(signer))
+                if bl.isSigned() and self._core._utils.validatePubKey(signer) and bl.isSigner(signer):
+                    validSig = True                    
+                bl.bheader['validSig'] = validSig
+                bl.bheader['meta'] = ''
+                retData = {'meta': bl.bheader, 'metadata': bl.bmetadata}
+            return json.dumps(retData)
+        else:
+            return bl.raw

onionr/blockimporter.py

@@ -0,0 +1,50 @@
+'''
+    Onionr - P2P Microblogging Platform & Social network
+
+    Import block data and save it
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import core, onionrexceptions, logger
+def importBlockFromData(content, coreInst):
+    retData = False
+
+    dataHash = coreInst._crypto.sha3Hash(content)
+
+    if coreInst._blacklist.inBlacklist(dataHash):
+        raise onionrexceptions.BlacklistedBlock('%s is a blacklisted block' % (dataHash,))
+
+    if not isinstance(coreInst, core.Core):
+        raise Exception("coreInst must be an Onionr core instance")
+
+    try:
+        content = content.encode()
+    except AttributeError:
+        pass
+
+    metas = coreInst._utils.getBlockMetadataFromData(content) # returns tuple(metadata, meta), meta is also in metadata
+    metadata = metas[0]
+    if coreInst._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid
+        if coreInst._crypto.verifyPow(content): # check if POW is enough/correct
+            logger.info('Block passed proof, saving.')
+            try:
+                blockHash = coreInst.setData(content)
+            except onionrexceptions.DiskAllocationReached:
+                pass
+            else:
+                coreInst.addToBlockDB(blockHash, dataSaved=True)
+                coreInst._utils.processBlockMetadata(blockHash) # caches block metadata values to block database
+                retData = True
+    return retData

onionr/communicator.py

@@ -0,0 +1,646 @@
+#!/usr/bin/env python3
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This file contains both the OnionrCommunicate class for communcating with peers
+    and code to operate as a daemon, getting commands from the command queue database (see core.Core.daemonQueue)
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sys, os, core, config, json, requests, time, logger, threading, base64, onionr, uuid, binascii
+from dependencies import secrets
+from utils import networkmerger
+import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block
+from communicatorutils import onionrdaemontools
+import onionrsockets, onionr, onionrproofs
+from communicatorutils import onionrcommunicatortimers, proxypicker
+
+OnionrCommunicatorTimers = onionrcommunicatortimers.OnionrCommunicatorTimers
+
+config.reload()
+class OnionrCommunicatorDaemon:
+    def __init__(self, onionrInst, proxyPort, developmentMode=config.get('general.dev_mode', False)):
+        onionrInst.communicatorInst = self
+        # configure logger and stuff
+        onionr.Onionr.setupConfig('data/', self = self)
+        self.proxyPort = proxyPort
+
+        self.isOnline = True # Assume we're connected to the internet
+
+        # list of timer instances
+        self.timers = []
+
+        # initialize core with Tor socks port being 3rd argument
+        self.proxyPort = proxyPort
+        self._core = onionrInst.onionrCore
+
+        self.blocksToUpload = []
+
+        # loop time.sleep delay in seconds
+        self.delay = 1
+
+        # lists of connected peers and peers we know we can't reach currently
+        self.onlinePeers = []
+        self.offlinePeers = []
+        self.cooldownPeer = {}
+        self.connectTimes = {}
+        self.peerProfiles = [] # list of peer's profiles (onionrpeers.PeerProfile instances)
+        self.newPeers = [] # Peers merged to us. Don't add to db until we know they're reachable
+
+        # amount of threads running by name, used to prevent too many
+        self.threadCounts = {}
+
+        # set true when shutdown command received
+        self.shutdown = False
+
+        # list of new blocks to download, added to when new block lists are fetched from peers
+        self.blockQueue = {}
+
+        # list of blocks currently downloading, avoid s
+        self.currentDownloading = []
+
+        # timestamp when the last online node was seen
+        self.lastNodeSeen = None
+
+        # Dict of time stamps for peer's block list lookup times, to avoid downloading full lists all the time
+        self.dbTimestamps = {}
+
+        # Clear the daemon queue for any dead messages
+        if os.path.exists(self._core.queueDB):
+            self._core.clearDaemonQueue()
+
+        # Loads in and starts the enabled plugins
+        plugins.reload()
+
+        # daemon tools are misc daemon functions, e.g. announce to online peers
+        # intended only for use by OnionrCommunicatorDaemon
+        self.daemonTools = onionrdaemontools.DaemonTools(self)
+
+        # time app started running for info/statistics purposes
+        self.startTime = self._core._utils.getEpoch()
+
+        if developmentMode:
+            OnionrCommunicatorTimers(self, self.heartbeat, 30)
+
+        # Set timers, function reference, seconds
+        # requiresPeer True means the timer function won't fire if we have no connected peers
+        peerPoolTimer = OnionrCommunicatorTimers(self, self.getOnlinePeers, 60, maxThreads=1)
+        OnionrCommunicatorTimers(self, self.runCheck, 2, maxThreads=1)
+        OnionrCommunicatorTimers(self, self.lookupBlocks, self._core.config.get('timers.lookupBlocks'), requiresPeer=True, maxThreads=1)
+        OnionrCommunicatorTimers(self, self.getBlocks, self._core.config.get('timers.getBlocks'), requiresPeer=True, maxThreads=2)
+        OnionrCommunicatorTimers(self, self.clearOfflinePeer, 58)
+        blockCleanupTimer = OnionrCommunicatorTimers(self, self.daemonTools.cleanOldBlocks, 65)
+        OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True)
+        OnionrCommunicatorTimers(self, self.daemonTools.cooldownPeer, 30, requiresPeer=True)
+        OnionrCommunicatorTimers(self, self.uploadBlock, 10, requiresPeer=True, maxThreads=1)
+        OnionrCommunicatorTimers(self, self.daemonCommands, 6, maxThreads=1)
+        OnionrCommunicatorTimers(self, self.detectAPICrash, 30, maxThreads=1)
+        deniableBlockTimer = OnionrCommunicatorTimers(self, self.daemonTools.insertDeniableBlock, 180, requiresPeer=True, maxThreads=1)
+
+        netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600)
+        if config.get('general.security_level') == 0:
+            announceTimer = OnionrCommunicatorTimers(self, self.daemonTools.announceNode, 3600, requiresPeer=True, maxThreads=1)
+            announceTimer.count = (announceTimer.frequency - 120)
+        else:
+            logger.debug('Will not announce node.')
+        cleanupTimer = OnionrCommunicatorTimers(self, self.peerCleanup, 300, requiresPeer=True)
+        forwardSecrecyTimer = OnionrCommunicatorTimers(self, self.daemonTools.cleanKeys, 15, maxThreads=1)
+
+        # set loop to execute instantly to load up peer pool (replaced old pool init wait)
+        peerPoolTimer.count = (peerPoolTimer.frequency - 1)
+        cleanupTimer.count = (cleanupTimer.frequency - 60)
+        deniableBlockTimer.count = (deniableBlockTimer.frequency - 175)
+        blockCleanupTimer.count = (blockCleanupTimer.frequency - 5)
+        #forwardSecrecyTimer.count = (forwardSecrecyTimer.frequency - 990)
+
+        if config.get('general.socket_servers'):
+            self.socketServer = threading.Thread(target=onionrsockets.OnionrSocketServer, args=(self._core,))
+            self.socketServer.start()
+            self.socketClient = onionrsockets.OnionrSocketClient(self._core)
+
+        # Main daemon loop, mainly for calling timers, don't do any complex operations here to avoid locking
+        try:
+            while not self.shutdown:
+                for i in self.timers:
+                    if self.shutdown:
+                        break
+                    i.processTimer()
+                time.sleep(self.delay)
+                # Debug to print out used FDs (regular and net)
+                #proc = psutil.Process()
+                #print(proc.open_files(), len(psutil.net_connections()))
+        except KeyboardInterrupt:
+            self.shutdown = True
+            pass
+
+        logger.info('Goodbye.')
+        self._core.killSockets = True
+        self._core._utils.localCommand('shutdown') # shutdown the api
+        time.sleep(0.5)
+
+    def lookupAdders(self):
+        '''Lookup new peer addresses'''
+        logger.info('Looking up new addresses...')
+        tryAmount = 1
+        newPeers = []
+        for i in range(tryAmount):
+            # Download new peer address list from random online peers
+            if len(newPeers) > 10000:
+                # Dont get new peers if we have too many queued up
+                break
+            peer = self.pickOnlinePeer()
+            newAdders = self.peerAction(peer, action='pex')
+            try:
+                newPeers = newAdders.split(',')
+            except AttributeError:
+                pass
+        else:
+            # Validate new peers are good format and not already in queue
+            invalid = []
+            for x in newPeers:
+                x = x.strip()
+                if not self._core._utils.validateID(x) or x in self.newPeers or x == self._core.hsAddress:
+                    invalid.append(x)
+            for x in invalid:
+                newPeers.remove(x)
+            self.newPeers.extend(newPeers)
+        self.decrementThreadCount('lookupAdders')
+
+    def lookupBlocks(self):
+        '''Lookup new blocks & add them to download queue'''
+        logger.info('Looking up new blocks...')
+        tryAmount = 2
+        newBlocks = ''
+        existingBlocks = self._core.getBlockList()
+        triedPeers = [] # list of peers we've tried this time around
+        maxBacklog = 1560 # Max amount of *new* block hashes to have already in queue, to avoid memory exhaustion
+        lastLookupTime = 0 # Last time we looked up a particular peer's list
+        for i in range(tryAmount):
+            listLookupCommand = 'getblocklist' # This is defined here to reset it each time
+            if len(self.blockQueue) >= maxBacklog:
+                break
+            if not self.isOnline:
+                break
+            # check if disk allocation is used
+            if self._core._utils.storageCounter.isFull():
+                logger.debug('Not looking up new blocks due to maximum amount of allowed disk space used')
+                break
+            peer = self.pickOnlinePeer() # select random online peer
+            # if we've already tried all the online peers this time around, stop
+            if peer in triedPeers:
+                if len(self.onlinePeers) == len(triedPeers):
+                    break
+                else:
+                    continue
+            triedPeers.append(peer)
+
+            # Get the last time we looked up a peer's stamp to only fetch blocks since then.
+            # Saved in memory only for privacy reasons
+            try:
+                lastLookupTime = self.dbTimestamps[peer]
+            except KeyError:
+                lastLookupTime = 0
+            else:
+                listLookupCommand += '?date=%s' % (lastLookupTime,)
+            try:
+                newBlocks = self.peerAction(peer, listLookupCommand) # get list of new block hashes
+            except Exception as error:
+                logger.warn('Could not get new blocks from %s.' % peer, error = error)
+                newBlocks = False
+            else:
+                self.dbTimestamps[peer] = self._core._utils.getRoundedEpoch(roundS=60)
+            if newBlocks != False:
+                # if request was a success
+                for i in newBlocks.split('\n'):
+                    if self._core._utils.validateHash(i):
+                        # if newline seperated string is valid hash
+                        if not i in existingBlocks:
+                            # if block does not exist on disk and is not already in block queue
+                            if i not in self.blockQueue:
+                                if onionrproofs.hashMeetsDifficulty(i) and not self._core._blacklist.inBlacklist(i):
+                                    if len(self.blockQueue) <= 1000000:
+                                        self.blockQueue[i] = [peer] # add blocks to download queue
+                            else:
+                                if peer not in self.blockQueue[i]:
+                                    if len(self.blockQueue[i]) < 10:
+                                        self.blockQueue[i].append(peer)
+        self.decrementThreadCount('lookupBlocks')
+        return
+
+    def getBlocks(self):
+        '''download new blocks in queue'''
+        for blockHash in list(self.blockQueue):
+            triedQueuePeers = [] # List of peers we've tried for a block
+            try:
+                blockPeers = list(self.blockQueue[blockHash])
+            except KeyError:
+                blockPeers = []
+            removeFromQueue = True
+            if self.shutdown or not self.isOnline:
+                # Exit loop if shutting down or offline
+                break
+            # Do not download blocks being downloaded or that are already saved (edge cases)
+            if blockHash in self.currentDownloading:
+                #logger.debug('Already downloading block %s...' % blockHash)
+                continue
+            if blockHash in self._core.getBlockList():
+                #logger.debug('Block %s is already saved.' % (blockHash,))
+                try:
+                    del self.blockQueue[blockHash]
+                except KeyError:
+                    pass
+                continue
+            if self._core._blacklist.inBlacklist(blockHash):
+                continue
+            if self._core._utils.storageCounter.isFull():
+                break
+            self.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block
+            if len(blockPeers) == 0:
+                peerUsed = self.pickOnlinePeer()
+            else:
+                blockPeers = self._core._crypto.randomShuffle(blockPeers)
+                peerUsed = blockPeers.pop(0)
+
+            if not self.shutdown and peerUsed.strip() != '':
+                logger.info("Attempting to download %s from %s..." % (blockHash[:12], peerUsed))
+            content = self.peerAction(peerUsed, 'getdata/' + blockHash) # block content from random peer (includes metadata)
+            if content != False and len(content) > 0:
+                try:
+                    content = content.encode()
+                except AttributeError:
+                    pass
+
+                realHash = self._core._crypto.sha3Hash(content)
+                try:
+                    realHash = realHash.decode() # bytes on some versions for some reason
+                except AttributeError:
+                    pass
+                if realHash == blockHash:
+                    content = content.decode() # decode here because sha3Hash needs bytes above
+                    metas = self._core._utils.getBlockMetadataFromData(content) # returns tuple(metadata, meta), meta is also in metadata
+                    metadata = metas[0]
+                    if self._core._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid, and verify nonce
+                        if self._core._crypto.verifyPow(content): # check if POW is enough/correct
+                            logger.info('Attempting to save block %s...' % blockHash[:12])
+                            try:
+                                self._core.setData(content)
+                            except onionrexceptions.DiskAllocationReached:
+                                logger.error('Reached disk allocation allowance, cannot save block %s.' % blockHash)
+                                removeFromQueue = False
+                            else:
+                                self._core.addToBlockDB(blockHash, dataSaved=True)
+                                self._core._utils.processBlockMetadata(blockHash) # caches block metadata values to block database
+                        else:
+                            logger.warn('POW failed for block %s.' % blockHash)
+                    else:
+                        if self._core._blacklist.inBlacklist(realHash):
+                            logger.warn('Block %s is blacklisted.' % (realHash,))
+                        else:
+                            logger.warn('Metadata for block %s is invalid.' % blockHash)
+                            self._core._blacklist.addToDB(blockHash)
+                else:
+                    # if block didn't meet expected hash
+                    tempHash = self._core._crypto.sha3Hash(content) # lazy hack, TODO use var
+                    try:
+                        tempHash = tempHash.decode()
+                    except AttributeError:
+                        pass
+                    # Punish peer for sharing invalid block (not always malicious, but is bad regardless)
+                    onionrpeers.PeerProfiles(peerUsed, self._core).addScore(-50)
+                    if tempHash != 'ed55e34cb828232d6c14da0479709bfa10a0923dca2b380496e6b2ed4f7a0253':
+                        # Dumb hack for 404 response from peer. Don't log it if 404 since its likely not malicious or a critical error.
+                        logger.warn('Block hash validation failed for ' + blockHash + ' got ' + tempHash)
+                    else:
+                        removeFromQueue = False # Don't remove from queue if 404
+                if removeFromQueue:
+                    try:
+                        del self.blockQueue[blockHash] # remove from block queue both if success or false
+                    except KeyError:
+                        pass
+            self.currentDownloading.remove(blockHash)
+        self.decrementThreadCount('getBlocks')
+        return
+
+    def pickOnlinePeer(self):
+        '''randomly picks peer from pool without bias (using secrets module)'''
+        retData = ''
+        while True:
+            peerLength = len(self.onlinePeers)
+            if peerLength <= 0:
+                break
+            try:
+                # get a random online peer, securely. May get stuck in loop if network is lost or if all peers in pool magically disconnect at once
+                retData = self.onlinePeers[self._core._crypto.secrets.randbelow(peerLength)]
+            except IndexError:
+                pass
+            else:
+                break
+        return retData
+
+    def decrementThreadCount(self, threadName):
+        '''Decrement amount of a thread name if more than zero, called when a function meant to be run in a thread ends'''
+        try:
+            if self.threadCounts[threadName] > 0:
+                self.threadCounts[threadName] -= 1
+        except KeyError:
+            pass
+
+    def clearOfflinePeer(self):
+        '''Removes the longest offline peer to retry later'''
+        try:
+            removed = self.offlinePeers.pop(0)
+        except IndexError:
+            pass
+        else:
+            logger.debug('Removed ' + removed + ' from offline list, will try them again.')
+        self.decrementThreadCount('clearOfflinePeer')
+
+    def getOnlinePeers(self):
+        '''
+            Manages the self.onlinePeers attribute list, connects to more peers if we have none connected
+        '''
+
+        logger.debug('Refreshing peer pool...')
+        maxPeers = int(config.get('peers.max_connect', 10))
+        needed = maxPeers - len(self.onlinePeers)
+
+        for i in range(needed):
+            if len(self.onlinePeers) == 0:
+                self.connectNewPeer(useBootstrap=True)
+            else:
+                self.connectNewPeer()
+
+            if self.shutdown:
+                break
+        else:
+            if len(self.onlinePeers) == 0:
+                logger.debug('Couldn\'t connect to any peers.' + (' Last node seen %s ago.' % self.daemonTools.humanReadableTime(time.time() - self.lastNodeSeen) if not self.lastNodeSeen is None else ''))
+            else:
+                self.lastNodeSeen = time.time()
+        self.decrementThreadCount('getOnlinePeers')
+
+    def addBootstrapListToPeerList(self, peerList):
+        '''
+            Add the bootstrap list to the peer list (no duplicates)
+        '''
+        for i in self._core.bootstrapList:
+            if i not in peerList and i not in self.offlinePeers and i != self._core.hsAddress and len(str(i).strip()) > 0:
+                peerList.append(i)
+                self._core.addAddress(i)
+
+    def connectNewPeer(self, peer='', useBootstrap=False):
+        '''Adds a new random online peer to self.onlinePeers'''
+        retData = False
+        tried = self.offlinePeers
+        if peer != '':
+            if self._core._utils.validateID(peer):
+                peerList = [peer]
+            else:
+                raise onionrexceptions.InvalidAddress('Will not attempt connection test to invalid address')
+        else:
+            peerList = self._core.listAdders()
+
+        mainPeerList = self._core.listAdders()
+        peerList = onionrpeers.getScoreSortedPeerList(self._core)
+
+        if len(peerList) < 8 or secrets.randbelow(4) == 3:
+            tryingNew = []
+            for x in self.newPeers:
+                if x not in peerList:
+                    peerList.append(x)
+                    tryingNew.append(x)
+            for i in tryingNew:
+                self.newPeers.remove(i)
+        
+        if len(peerList) == 0 or useBootstrap:
+            # Avoid duplicating bootstrap addresses in peerList
+            self.addBootstrapListToPeerList(peerList)
+
+        for address in peerList:
+            if not config.get('tor.v3onions') and len(address) == 62:
+                continue
+            if address == self._core.hsAddress:
+                continue
+            if len(address) == 0 or address in tried or address in self.onlinePeers or address in self.cooldownPeer:
+                continue
+            if self.shutdown:
+                return
+            if self.peerAction(address, 'ping') == 'pong!':
+                time.sleep(0.1)
+                if address not in mainPeerList:
+                    networkmerger.mergeAdders(address, self._core)
+                if address not in self.onlinePeers:
+                    logger.info('Connected to ' + address)
+                    self.onlinePeers.append(address)
+                    self.connectTimes[address] = self._core._utils.getEpoch()
+                retData = address
+
+                # add peer to profile list if they're not in it
+                for profile in self.peerProfiles:
+                    if profile.address == address:
+                        break
+                else:
+                    self.peerProfiles.append(onionrpeers.PeerProfiles(address, self._core))
+                break
+            else:
+                tried.append(address)
+                logger.debug('Failed to connect to ' + address)
+        return retData
+
+    def removeOnlinePeer(self, peer):
+        '''Remove an online peer'''
+        try:
+            del self.connectTimes[peer]
+        except KeyError:
+            pass
+        try:
+            del self.dbTimestamps[peer]
+        except KeyError:
+            pass
+        try:
+            self.onlinePeers.remove(peer)
+        except ValueError:
+            pass
+
+    def peerCleanup(self):
+        '''This just calls onionrpeers.cleanupPeers, which removes dead or bad peers (offline too long, too slow)'''
+        onionrpeers.peerCleanup(self._core)
+        self.decrementThreadCount('peerCleanup')
+
+    def printOnlinePeers(self):
+        '''logs online peer list'''
+        if len(self.onlinePeers) == 0:
+            logger.warn('No online peers')
+        else:
+            logger.info('Online peers:')
+            for i in self.onlinePeers:
+                score = str(self.getPeerProfileInstance(i).score)
+                logger.info(i + ', score: ' + score)
+
+    def peerAction(self, peer, action, data='', returnHeaders=False):
+        '''Perform a get request to a peer'''
+        if len(peer) == 0:
+            return False
+        #logger.debug('Performing ' + action + ' with ' + peer + ' on port ' + str(self.proxyPort))
+        url = 'http://%s/%s' % (peer, action)
+        if len(data) > 0:
+            url += '&data=' + data
+
+        self._core.setAddressInfo(peer, 'lastConnectAttempt', self._core._utils.getEpoch()) # mark the time we're trying to request this peer
+
+        retData = self._core._utils.doGetRequest(url, port=self.proxyPort)
+        # if request failed, (error), mark peer offline
+        if retData == False:
+            try:
+                self.getPeerProfileInstance(peer).addScore(-10)
+                self.removeOnlinePeer(peer)
+                if action != 'ping':
+                    self.getOnlinePeers() # Will only add a new peer to pool if needed
+            except ValueError:
+                pass
+        else:
+            self._core.setAddressInfo(peer, 'lastConnect', self._core._utils.getEpoch())
+            self.getPeerProfileInstance(peer).addScore(1)
+        return retData # If returnHeaders, returns tuple of data, headers. if not, just data string
+
+    def getPeerProfileInstance(self, peer):
+        '''Gets a peer profile instance from the list of profiles, by address name'''
+        for i in self.peerProfiles:
+            # if the peer's profile is already loaded, return that
+            if i.address == peer:
+                retData = i
+                break
+        else:
+            # if the peer's profile is not loaded, return a new one. connectNewPeer adds it the list on connect
+            retData = onionrpeers.PeerProfiles(peer, self._core)
+        return retData
+
+    def getUptime(self):
+        return self._core._utils.getEpoch() - self.startTime
+
+    def heartbeat(self):
+        '''Show a heartbeat debug message'''
+        logger.debug('Heartbeat. Node running for %s.' % self.daemonTools.humanReadableTime(self.getUptime()))
+        self.decrementThreadCount('heartbeat')
+
+    def daemonCommands(self):
+        '''
+            Process daemon commands from daemonQueue
+        '''
+        cmd = self._core.daemonQueue()
+        response = ''
+        if cmd is not False:
+            events.event('daemon_command', onionr = None, data = {'cmd' : cmd})
+            if cmd[0] == 'shutdown':
+                self.shutdown = True
+            elif cmd[0] == 'announceNode':
+                if len(self.onlinePeers) > 0:
+                    self.announce(cmd[1])
+                else:
+                    logger.debug("No nodes connected. Will not introduce node.")
+            elif cmd[0] == 'runCheck': # deprecated
+                logger.debug('Status check; looks good.')
+                open(self._core.dataDir + '.runcheck', 'w+').close()
+            elif cmd[0] == 'connectedPeers':
+                response = '\n'.join(list(self.onlinePeers)).strip()
+                if response == '':
+                    response = 'none'
+            elif cmd[0] == 'localCommand':
+                response = self._core._utils.localCommand(cmd[1])
+            elif cmd[0] == 'pex':
+                for i in self.timers:
+                    if i.timerFunction.__name__ == 'lookupAdders':
+                        i.count = (i.frequency - 1)
+            elif cmd[0] == 'uploadBlock':
+                self.blocksToUpload.append(cmd[1])
+            elif cmd[0] == 'startSocket':
+                # Create our own socket server
+                socketInfo = json.loads(cmd[1])
+                socketInfo['id'] = uuid.uuid4()
+                self._core.startSocket = socketInfo
+            elif cmd[0] == 'addSocket':
+                # Socket server was created for us
+                socketInfo = json.loads(cmd[1])
+                peer = socketInfo['peer']
+                reason = socketInfo['reason']
+                threading.Thread(target=self.socketClient.startSocket, args=(peer, reason)).start()
+            else:
+                logger.info('Recieved daemonQueue command:' + cmd[0])
+
+            if cmd[0] not in ('', None):
+                if response != '':
+                    self._core._utils.localCommand('queueResponseAdd/' + cmd[4], post=True, postData={'data': response})
+            response = ''
+
+        self.decrementThreadCount('daemonCommands')
+
+    def uploadBlock(self):
+        '''Upload our block to a few peers'''
+        # when inserting a block, we try to upload it to a few peers to add some deniability
+        triedPeers = []
+        finishedUploads = []
+        self.blocksToUpload = self._core._crypto.randomShuffle(self.blocksToUpload)
+        if len(self.blocksToUpload) != 0:
+            for bl in self.blocksToUpload:
+                if not self._core._utils.validateHash(bl):
+                    logger.warn('Requested to upload invalid block')
+                    self.decrementThreadCount('uploadBlock')
+                    return
+                for i in range(min(len(self.onlinePeers), 6)):
+                    peer = self.pickOnlinePeer()
+                    if peer in triedPeers:
+                        continue
+                    triedPeers.append(peer)
+                    url = 'http://' + peer + '/upload'
+                    data = {'block': block.Block(bl).getRaw()}
+                    proxyType = proxypicker.pick_proxy(peer)
+                    logger.info("Uploading block to " + peer)
+                    if not self._core._utils.doPostRequest(url, data=data, proxyType=proxyType) == False:
+                        self._core._utils.localCommand('waitforshare/' + bl, post=True)
+                        finishedUploads.append(bl)
+        for x in finishedUploads:
+            try:
+                self.blocksToUpload.remove(x)
+            except ValueError:
+                pass
+        self.decrementThreadCount('uploadBlock')
+
+    def announce(self, peer):
+        '''Announce to peers our address'''
+        if self.daemonTools.announceNode() == False:
+            logger.warn('Could not introduce node.')
+
+    def detectAPICrash(self):
+        '''exit if the api server crashes/stops'''
+        if self._core._utils.localCommand('ping', silent=False) not in ('pong', 'pong!'):
+            for i in range(8):
+                if self._core._utils.localCommand('ping') in ('pong', 'pong!') or self.shutdown:
+                    break # break for loop
+                time.sleep(1)
+            else:
+                # This executes if the api is NOT detected to be running
+                events.event('daemon_crash', onionr = None, data = {})
+                logger.error('Daemon detected API crash (or otherwise unable to reach API after long time), stopping...')
+                self.shutdown = True
+        self.decrementThreadCount('detectAPICrash')
+
+    def runCheck(self):
+        if self.daemonTools.runCheck():
+            logger.debug('Status check; looks good.')
+
+        self.decrementThreadCount('runCheck')
+
+def startCommunicator(onionrInst, proxyPort):
+    OnionrCommunicatorDaemon(onionrInst, proxyPort)

onionr/communicatorutils/onionrcommunicatortimers.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This file contains timer control for the communicator
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import threading, onionrexceptions, logger
+class OnionrCommunicatorTimers:
+    def __init__(self, daemonInstance, timerFunction, frequency, makeThread=True, threadAmount=1, maxThreads=5, requiresPeer=False):
+        self.timerFunction = timerFunction
+        self.frequency = frequency
+        self.threadAmount = threadAmount
+        self.makeThread = makeThread
+        self.requiresPeer = requiresPeer
+        self.daemonInstance = daemonInstance
+        self.maxThreads = maxThreads
+        self._core = self.daemonInstance._core
+
+        self.daemonInstance.timers.append(self)
+        self.count = 0
+
+    def processTimer(self):
+
+        # mark how many instances of a thread we have (decremented at thread end)
+        try:
+            self.daemonInstance.threadCounts[self.timerFunction.__name__]
+        except KeyError:
+            self.daemonInstance.threadCounts[self.timerFunction.__name__] = 0
+
+        # execute thread if it is time, and we are not missing *required* online peer
+        if self.count == self.frequency and not self.daemonInstance.shutdown:
+            try:
+                if self.requiresPeer and len(self.daemonInstance.onlinePeers) == 0:
+                    raise onionrexceptions.OnlinePeerNeeded
+            except onionrexceptions.OnlinePeerNeeded:
+                pass
+            else:
+                if self.makeThread:
+                    for i in range(self.threadAmount):
+                        if self.daemonInstance.threadCounts[self.timerFunction.__name__] >= self.maxThreads:
+                            logger.debug('%s is currently using the maximum number of threads, not starting another.' % self.timerFunction.__name__)
+                        else:
+                            self.daemonInstance.threadCounts[self.timerFunction.__name__] += 1
+                            newThread = threading.Thread(target=self.timerFunction)
+                            newThread.start()
+                else:
+                    self.timerFunction()
+            self.count = -1 # negative 1 because its incremented at bottom
+        self.count += 1

onionr/communicatorutils/onionrdaemontools.py

@@ -0,0 +1,214 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    Contains the CommunicatorUtils class which contains useful functions for the communicator daemon
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+import onionrexceptions, onionrpeers, onionrproofs, logger
+import base64, sqlite3, os
+from dependencies import secrets
+from utils import netutils
+from onionrusers import onionrusers
+
+class DaemonTools:
+    '''
+        Class intended for use by Onionr Communicator
+    '''
+    def __init__(self, daemon):
+            self.daemon = daemon
+            self.announceProgress = {}
+            self.announceCache = {}
+
+    def announceNode(self):
+        '''Announce our node to our peers'''
+        retData = False
+        announceFail = False
+        
+        # Do not let announceCache get too large
+        if len(self.announceCache) >= 10000:
+            self.announceCache.popitem()
+
+        if self.daemon._core.config.get('general.security_level', 0) == 0:
+            # Announce to random online peers
+            for i in self.daemon.onlinePeers:
+                if not i in self.announceCache and not i in self.announceProgress:
+                    peer = i
+                    break
+            else:
+                peer = self.daemon.pickOnlinePeer()
+
+            for x in range(1):
+                if x == 1 and self.daemon._core.config.get('i2p.host'):
+                    ourID = self.daemon._core.config.get('i2p.own_addr').strip()
+                else:
+                    ourID = self.daemon._core.hsAddress.strip()
+
+                url = 'http://' + peer + '/announce'
+                data = {'node': ourID}
+
+                combinedNodes = ourID + peer
+                if ourID != 1:
+                    #TODO: Extend existingRand for i2p
+                    existingRand = self.daemon._core.getAddressInfo(peer, 'powValue')
+                    if type(existingRand) is type(None):
+                        existingRand = ''
+
+                if peer in self.announceCache:
+                    data['random'] = self.announceCache[peer]
+                elif len(existingRand) > 0:
+                    data['random'] = existingRand
+                else:
+                    self.announceProgress[peer] = True
+                    proof = onionrproofs.DataPOW(combinedNodes, forceDifficulty=4)
+                    del self.announceProgress[peer]
+                    try:
+                        data['random'] = base64.b64encode(proof.waitForResult()[1])
+                    except TypeError:
+                        # Happens when we failed to produce a proof
+                        logger.error("Failed to produce a pow for announcing to " + peer)
+                        announceFail = True
+                    else:
+                        self.announceCache[peer] = data['random']
+                if not announceFail:
+                    logger.info('Announcing node to ' + url)
+                    if self.daemon._core._utils.doPostRequest(url, data) == 'Success':
+                        logger.info('Successfully introduced node to ' + peer)
+                        retData = True
+                        self.daemon._core.setAddressInfo(peer, 'introduced', 1)
+                        self.daemon._core.setAddressInfo(peer, 'powValue', data['random'])
+        self.daemon.decrementThreadCount('announceNode')
+        return retData
+
+    def netCheck(self):
+        '''Check if we are connected to the internet or not when we can't connect to any peers'''
+        if len(self.daemon.onlinePeers) == 0:
+            if not netutils.checkNetwork(self.daemon._core._utils, torPort=self.daemon.proxyPort):
+                if not self.daemon.shutdown:
+                    logger.warn('Network check failed, are you connected to the internet?')
+                self.daemon.isOnline = False
+            else:
+                self.daemon.isOnline = True
+        self.daemon.decrementThreadCount('netCheck')
+
+    def cleanOldBlocks(self):
+        '''Delete old blocks if our disk allocation is full/near full, and also expired blocks'''
+
+        # Delete expired blocks
+        for bHash in self.daemon._core.getExpiredBlocks():
+            self.daemon._core._blacklist.addToDB(bHash)
+            self.daemon._core.removeBlock(bHash)
+            logger.info('Deleted block: %s' % (bHash,))
+
+        while self.daemon._core._utils.storageCounter.isFull():
+            oldest = self.daemon._core.getBlockList()[0]
+            self.daemon._core._blacklist.addToDB(oldest)
+            self.daemon._core.removeBlock(oldest)
+            logger.info('Deleted block: %s' % (oldest,))
+
+        self.daemon.decrementThreadCount('cleanOldBlocks')
+
+    def cleanKeys(self):
+        '''Delete expired forward secrecy keys'''
+        conn = sqlite3.connect(self.daemon._core.peerDB, timeout=10)
+        c = conn.cursor()
+        time = self.daemon._core._utils.getEpoch()
+        deleteKeys = []
+
+        for entry in c.execute("SELECT * FROM forwardKeys WHERE expire <= ?", (time,)):
+            logger.debug('Forward key: %s' % entry[1])
+            deleteKeys.append(entry[1])
+
+        for key in deleteKeys:
+            logger.debug('Deleting forward key %s' % key)
+            c.execute("DELETE from forwardKeys where forwardKey = ?", (key,))
+        conn.commit()
+        conn.close()
+
+        onionrusers.deleteExpiredKeys(self.daemon._core)
+
+        self.daemon.decrementThreadCount('cleanKeys')
+
+    def cooldownPeer(self):
+        '''Randomly add an online peer to cooldown, so we can connect a new one'''
+        onlinePeerAmount = len(self.daemon.onlinePeers)
+        minTime = 300
+        cooldownTime = 600
+        toCool = ''
+        tempConnectTimes = dict(self.daemon.connectTimes)
+
+        # Remove peers from cooldown that have been there long enough
+        tempCooldown = dict(self.daemon.cooldownPeer)
+        for peer in tempCooldown:
+            if (self.daemon._core._utils.getEpoch() - tempCooldown[peer]) >= cooldownTime:
+                del self.daemon.cooldownPeer[peer]
+
+        # Cool down a peer, if we have max connections alive for long enough
+        if onlinePeerAmount >= self.daemon._core.config.get('peers.max_connect', 10, save = True):
+            finding = True
+
+            while finding:
+                try:
+                    toCool = min(tempConnectTimes, key=tempConnectTimes.get)
+                    if (self.daemon._core._utils.getEpoch() - tempConnectTimes[toCool]) < minTime:
+                        del tempConnectTimes[toCool]
+                    else:
+                        finding = False
+                except ValueError:
+                    break
+            else:
+                self.daemon.removeOnlinePeer(toCool)
+                self.daemon.cooldownPeer[toCool] = self.daemon._core._utils.getEpoch()
+
+        self.daemon.decrementThreadCount('cooldownPeer')
+
+    def runCheck(self):
+        if os.path.isfile(self.daemon._core.dataDir + '.runcheck'):
+            os.remove(self.daemon._core.dataDir + '.runcheck')
+            return True
+
+        return False
+
+    def humanReadableTime(self, seconds):
+        build = ''
+
+        units = {
+            'year' : 31557600,
+            'month' : (31557600 / 12),
+            'day' : 86400,
+            'hour' : 3600,
+            'minute' : 60,
+            'second' : 1
+        }
+
+        for unit in units:
+            amnt_unit = int(seconds / units[unit])
+            if amnt_unit >= 1:
+                seconds -= amnt_unit * units[unit]
+                build += '%s %s' % (amnt_unit, unit) + ('s' if amnt_unit != 1 else '') + ' '
+
+        return build.strip()
+
+    def insertDeniableBlock(self):
+        '''Insert a fake block in order to make it more difficult to track real blocks'''
+        fakePeer = ''
+        chance = 10
+        if secrets.randbelow(chance) == (chance - 1):
+            fakePeer = 'OVPCZLOXD6DC5JHX4EQ3PSOGAZ3T24F75HQLIUZSDSMYPEOXCPFA===='
+            data = secrets.token_hex(secrets.randbelow(500) + 1)
+            self.daemon._core.insertBlock(data, header='pm', encryptType='asym', asymPeer=fakePeer, meta={'subject': 'foo'})
+        self.daemon.decrementThreadCount('insertDeniableBlock')
+        return

onionr/communicatorutils/proxypicker.py

@@ -0,0 +1,25 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    Just picks a proxy
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+def pick_proxy(peer_address):
+    if peer_address.endswith('.onion'):
+        return 'tor'
+    elif peer_address.endswith('.i2p'):
+        return 'i2p'

onionr/config.py

@@ -0,0 +1,158 @@
+'''
+    Onionr - P2P Microblogging Platform & Social network
+
+    This file deals with configuration management.
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+import os, json, logger
+
+# set data dir
+dataDir = os.environ.get('ONIONR_HOME', os.environ.get('DATA_DIR', 'data/'))
+if not dataDir.endswith('/'):
+    dataDir += '/'
+
+_configfile = os.path.abspath(dataDir + 'config.json')
+_config = {}
+
+def get(key, default = None, save = False):
+    '''
+        Gets the key from configuration, or returns `default`
+    '''
+
+    key = str(key).split('.')
+    data = _config
+
+    last = key.pop()
+
+    for item in key:
+        if (not item in data) or (not type(data[item]) == dict):
+            return default
+        data = data[item]
+
+    if not last in data:
+        if save:
+            set(key, default, savefile = True)
+        return default
+
+    return data[last]
+
+def set(key, value = None, savefile = False):
+    '''
+        Sets the key in configuration to `value`
+    '''
+
+    global _config
+
+    key = str(key).split('.')
+    data = _config
+
+    last = key.pop()
+
+    for item in key:
+        if (not item in data) or (not type(data[item]) == dict):
+            data[item] = dict()
+        data = data[item]
+
+    if value is None:
+        del data[last]
+    else:
+        data[last] = value
+
+    if savefile:
+        save()
+
+def is_set(key):
+    key = str(key).split('.')
+    data = _config
+
+    last = key.pop()
+
+    for item in key:
+        if (not item in data) or (not type(data[item]) == dict):
+            return False
+        data = data[item]
+
+    if not last in data:
+        return False
+
+    return True
+
+def check():
+    '''
+        Checks if the configuration file exists, creates it if not
+    '''
+
+    try:
+        if not os.path.exists(os.path.dirname(get_config_file())):
+            os.path.mkdirs(os.path.dirname(get_config_file()))
+        if not os.path.isfile(get_config_file()):
+            open(get_config_file(), 'a', encoding="utf8").close()
+            save()
+    except:
+        pass
+        #logger.debug('Failed to check configuration file.')
+
+def save():
+    '''
+        Saves the configuration data to the configuration file
+    '''
+
+    check()
+    try:
+        with open(get_config_file(), 'w', encoding="utf8") as configfile:
+            json.dump(get_config(), configfile, indent=2)
+    except:
+        logger.warn('Failed to write to configuration file.')
+
+def reload():
+    '''
+        Reloads the configuration data in memory from the file
+    '''
+
+    check()
+    try:
+        with open(get_config_file(), 'r', encoding="utf8") as configfile:
+            set_config(json.loads(configfile.read()))
+    except:
+        pass
+        #logger.debug('Failed to parse configuration file.')
+
+def get_config():
+    '''
+        Gets the entire configuration as an array
+    '''
+    return _config
+
+def set_config(config):
+    '''
+        Sets the configuration to the array in arguments
+    '''
+    global _config
+    _config = config
+
+def get_config_file():
+    '''
+        Returns the absolute path to the configuration file
+    '''
+    return _configfile
+
+def set_config_file(configfile):
+    '''
+        Sets the path to the configuration file
+    '''
+    global _configfile
+    _configfile = os.abs.abspath(configfile)

onionr/core.py

@@ -0,0 +1,838 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    Core Onionr library, useful for external programs. Handles peer & data processing
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sqlite3, os, sys, time, math, base64, tarfile, nacl, logger, json, netcontroller, math, config, uuid
+from onionrblockapi import Block
+import deadsimplekv as simplekv
+import onionrutils, onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions
+import onionrblacklist
+from onionrusers import onionrusers
+import dbcreator, onionrstorage, serializeddata, subprocesspow
+from etc import onionrvalues
+
+if sys.version_info < (3, 6):
+    try:
+        import sha3
+    except ModuleNotFoundError:
+        logger.fatal('On Python 3 versions prior to 3.6.x, you need the sha3 module')
+        sys.exit(1)
+
+class Core:
+    def __init__(self, torPort=0):
+        '''
+            Initialize Core Onionr library
+        '''
+
+        # set data dir
+        self.dataDir = os.environ.get('ONIONR_HOME', os.environ.get('DATA_DIR', 'data/'))
+        if not self.dataDir.endswith('/'):
+            self.dataDir += '/'
+
+        try:
+            self.onionrInst = None
+            self.queueDB = self.dataDir + 'queue.db'
+            self.peerDB = self.dataDir + 'peers.db'
+            self.blockDB = self.dataDir + 'blocks.db'
+            self.blockDataLocation = self.dataDir + 'blocks/'
+            self.blockDataDB = self.blockDataLocation + 'block-data.db'
+            self.publicApiHostFile = self.dataDir + 'public-host.txt'
+            self.privateApiHostFile = self.dataDir + 'private-host.txt'
+            self.addressDB = self.dataDir + 'address.db'
+            self.hsAddress = ''
+            self.i2pAddress = config.get('i2p.own_addr', None)
+            self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt'
+            self.bootstrapList = []
+            self.requirements = onionrvalues.OnionrValues()
+            self.torPort = torPort
+            self.dataNonceFile = self.dataDir + 'block-nonces.dat'
+            self.dbCreate = dbcreator.DBCreator(self)
+            self.forwardKeysFile = self.dataDir + 'forward-keys.db'
+            self.keyStore = simplekv.DeadSimpleKV(self.dataDir + 'cachedstorage.dat', refresh_seconds=5)
+
+            # Socket data, defined here because of multithreading constraints with gevent
+            self.killSockets = False
+            self.startSocket = {}
+            self.socketServerConnData = {}
+            self.socketReasons = {}
+            self.socketServerResponseData = {}
+
+            self.usageFile = self.dataDir + 'disk-usage.txt'
+            self.config = config
+
+            self.maxBlockSize = 10000000 # max block size in bytes
+
+            if not os.path.exists(self.dataDir):
+                os.mkdir(self.dataDir)
+            if not os.path.exists(self.dataDir + 'blocks/'):
+                os.mkdir(self.dataDir + 'blocks/')
+            if not os.path.exists(self.blockDB):
+                self.createBlockDB()
+            if not os.path.exists(self.forwardKeysFile):
+                self.dbCreate.createForwardKeyDB()
+            if not os.path.exists(self.peerDB):
+                self.createPeerDB()
+            if not os.path.exists(self.addressDB):
+                self.createAddressDB()
+
+            if os.path.exists(self.dataDir + '/hs/hostname'):
+                with open(self.dataDir + '/hs/hostname', 'r') as hs:
+                    self.hsAddress = hs.read().strip()
+
+            # Load bootstrap address list
+            if os.path.exists(self.bootstrapFileLocation):
+                with open(self.bootstrapFileLocation, 'r') as bootstrap:
+                    bootstrap = bootstrap.read()
+                for i in bootstrap.split('\n'):
+                    self.bootstrapList.append(i)
+            else:
+                logger.warn('Warning: address bootstrap file not found ' + self.bootstrapFileLocation)
+
+            self._utils = onionrutils.OnionrUtils(self)
+            # Initialize the crypto object
+            self._crypto = onionrcrypto.OnionrCrypto(self)
+            self._blacklist = onionrblacklist.OnionrBlackList(self)
+            self.serializer = serializeddata.SerializedData(self)
+
+        except Exception as error:
+            logger.error('Failed to initialize core Onionr library.', error=error)
+            logger.fatal('Cannot recover from error.')
+            sys.exit(1)
+        return
+
+    def refreshFirstStartVars(self):
+        '''
+            Hack to refresh some vars which may not be set on first start
+        '''
+        if os.path.exists(self.dataDir + '/hs/hostname'):
+            with open(self.dataDir + '/hs/hostname', 'r') as hs:
+                self.hsAddress = hs.read().strip()
+
+    def addPeer(self, peerID, name=''):
+        '''
+            Adds a public key to the key database (misleading function name)
+        '''
+        assert peerID not in self.listPeers()
+
+        # This function simply adds a peer to the DB
+        if not self._utils.validatePubKey(peerID):
+            return False
+
+        events.event('pubkey_add', data = {'key': peerID}, onionr = None)
+
+        conn = sqlite3.connect(self.peerDB, timeout=30)
+        hashID = self._crypto.pubKeyHashID(peerID)
+        c = conn.cursor()
+        t = (peerID, name, 'unknown', hashID, 0)
+
+        for i in c.execute("SELECT * FROM peers WHERE id = ?;", (peerID,)):
+            try:
+                if i[0] == peerID:
+                    conn.close()
+                    return False
+            except ValueError:
+                pass
+            except IndexError:
+                pass
+        c.execute('INSERT INTO peers (id, name, dateSeen, hashID, trust) VALUES(?, ?, ?, ?, ?);', t)
+        conn.commit()
+        conn.close()
+
+        return True
+
+    def addAddress(self, address):
+        '''
+            Add an address to the address database (only tor currently)
+        '''
+
+        if address == config.get('i2p.ownAddr', None) or address == self.hsAddress:
+            return False
+        if type(address) is None or len(address) == 0:
+            return False
+        if self._utils.validateID(address):
+            conn = sqlite3.connect(self.addressDB, timeout=30)
+            c = conn.cursor()
+            # check if address is in database
+            # this is safe to do because the address is validated above, but we strip some chars here too just in case
+            address = address.replace('\'', '').replace(';', '').replace('"', '').replace('\\', '')
+            for i in c.execute("SELECT * FROM adders WHERE address = ?;", (address,)):
+                try:
+                    if i[0] == address:
+                        conn.close()
+                        return False
+                except ValueError:
+                    pass
+                except IndexError:
+                    pass
+
+            t = (address, 1)
+            c.execute('INSERT INTO adders (address, type) VALUES(?, ?);', t)
+            conn.commit()
+            conn.close()
+
+            events.event('address_add', data = {'address': address}, onionr = None)
+
+            return True
+        else:
+            #logger.debug('Invalid ID: %s' % address)
+            return False
+
+    def removeAddress(self, address):
+        '''
+            Remove an address from the address database
+        '''
+
+        if self._utils.validateID(address):
+            conn = sqlite3.connect(self.addressDB, timeout=30)
+            c = conn.cursor()
+            t = (address,)
+            c.execute('Delete from adders where address=?;', t)
+            conn.commit()
+            conn.close()
+
+            events.event('address_remove', data = {'address': address}, onionr = None)
+            return True
+        else:
+            return False
+
+    def removeBlock(self, block):
+        '''
+            remove a block from this node (does not automatically blacklist)
+
+            **You may want blacklist.addToDB(blockHash)
+        '''
+
+        if self._utils.validateHash(block):
+            conn = sqlite3.connect(self.blockDB, timeout=30)
+            c = conn.cursor()
+            t = (block,)
+            c.execute('Delete from hashes where hash=?;', t)
+            conn.commit()
+            conn.close()
+            dataSize = sys.getsizeof(onionrstorage.getData(self, block))
+            self._utils.storageCounter.removeBytes(dataSize)
+
+    def createAddressDB(self):
+        '''
+            Generate the address database
+        '''
+        self.dbCreate.createAddressDB()
+
+    def createPeerDB(self):
+        '''
+            Generate the peer sqlite3 database and populate it with the peers table.
+        '''
+        self.dbCreate.createPeerDB()
+
+    def createBlockDB(self):
+        '''
+            Create a database for blocks
+        '''
+        self.dbCreate.createBlockDB()
+
+    def addToBlockDB(self, newHash, selfInsert=False, dataSaved=False):
+        '''
+            Add a hash value to the block db
+
+            Should be in hex format!
+        '''
+
+        if not os.path.exists(self.blockDB):
+            raise Exception('Block db does not exist')
+        if self._utils.hasBlock(newHash):
+            return
+        conn = sqlite3.connect(self.blockDB, timeout=30)
+        c = conn.cursor()
+        currentTime = self._utils.getEpoch() + self._crypto.secrets.randbelow(301)
+        if selfInsert or dataSaved:
+            selfInsert = 1
+        else:
+            selfInsert = 0
+        data = (newHash, currentTime, '', selfInsert)
+        c.execute('INSERT INTO hashes (hash, dateReceived, dataType, dataSaved) VALUES(?, ?, ?, ?);', data)
+        conn.commit()
+        conn.close()
+
+        return
+
+    def getData(self, hash):
+        '''
+            Simply return the data associated to a hash
+        '''
+
+        data = onionrstorage.getData(self, hash)
+
+        return data
+
+    def setData(self, data):
+        '''
+            Set the data assciated with a hash
+        '''
+
+        data = data
+        dataSize = sys.getsizeof(data)
+
+        if not type(data) is bytes:
+            data = data.encode()
+
+        dataHash = self._crypto.sha3Hash(data)
+
+        if type(dataHash) is bytes:
+            dataHash = dataHash.decode()
+        blockFileName = self.blockDataLocation + dataHash + '.dat'
+        if os.path.exists(blockFileName):
+            pass # TODO: properly check if block is already saved elsewhere
+            #raise Exception("Data is already set for " + dataHash)
+        else:
+            if self._utils.storageCounter.addBytes(dataSize) != False:
+                onionrstorage.store(self, data, blockHash=dataHash)
+                conn = sqlite3.connect(self.blockDB, timeout=30)
+                c = conn.cursor()
+                c.execute("UPDATE hashes SET dataSaved=1 WHERE hash = ?;", (dataHash,))
+                conn.commit()
+                conn.close()
+                with open(self.dataNonceFile, 'a') as nonceFile:
+                    nonceFile.write(dataHash + '\n')
+            else:
+                raise onionrexceptions.DiskAllocationReached
+
+        return dataHash
+
+    def daemonQueue(self):
+        '''
+            Gives commands to the communication proccess/daemon by reading an sqlite3 database
+
+            This function intended to be used by the client. Queue to exchange data between "client" and server.
+        '''
+
+        retData = False
+        if not os.path.exists(self.queueDB):
+            self.dbCreate.createDaemonDB()
+        else:
+            conn = sqlite3.connect(self.queueDB, timeout=30)
+            c = conn.cursor()
+            try:
+                for row in c.execute('SELECT command, data, date, min(ID), responseID FROM commands group by id'):
+                    retData = row
+                    break
+            except sqlite3.OperationalError:
+                self.dbCreate.createDaemonDB()
+            else:
+                if retData != False:
+                    c.execute('DELETE FROM commands WHERE id=?;', (retData[3],))
+            conn.commit()
+            conn.close()
+
+        events.event('queue_pop', data = {'data': retData}, onionr = None)
+
+        return retData
+
+    def daemonQueueAdd(self, command, data='', responseID=''):
+        '''
+            Add a command to the daemon queue, used by the communication daemon (communicator.py)
+        '''
+
+        retData = True
+
+        date = self._utils.getEpoch()
+        conn = sqlite3.connect(self.queueDB, timeout=30)
+        c = conn.cursor()
+        t = (command, data, date, responseID)
+
+        try:
+            c.execute('INSERT INTO commands (command, data, date, responseID) VALUES(?, ?, ?, ?)', t)
+            conn.commit()
+        except sqlite3.OperationalError:
+            retData = False
+            self.daemonQueue()
+        events.event('queue_push', data = {'command': command, 'data': data}, onionr = None)
+        conn.close()
+        return retData
+
+    def daemonQueueGetResponse(self, responseID=''):
+        '''
+            Get a response sent by communicator to the API, by requesting to the API
+        '''
+        assert len(responseID) > 0
+        resp = self._utils.localCommand('queueResponse/' + responseID)
+        return resp
+
+    def daemonQueueWaitForResponse(self, responseID='', checkFreqSecs=1):
+        resp = 'failure'
+        while resp == 'failure':
+            resp = self.daemonQueueGetResponse(responseID)
+            time.sleep(1)
+        return resp
+
+    def daemonQueueSimple(self, command, data='', checkFreqSecs=1):
+        '''
+        A simplified way to use the daemon queue. Will register a command (with optional data) and wait, return the data
+        Not always useful, but saves time + LOC in some cases.
+        This is a blocking function, so be careful.
+        '''
+        responseID = str(uuid.uuid4()) # generate unique response ID
+        self.daemonQueueAdd(command, data=data, responseID=responseID)
+        return self.daemonQueueWaitForResponse(responseID, checkFreqSecs)
+
+    def clearDaemonQueue(self):
+        '''
+            Clear the daemon queue (somewhat dangerous)
+        '''
+        conn = sqlite3.connect(self.queueDB, timeout=30)
+        c = conn.cursor()
+
+        try:
+            c.execute('DELETE FROM commands;')
+            conn.commit()
+        except:
+            pass
+
+        conn.close()
+        events.event('queue_clear', onionr = None)
+
+        return
+
+    def listAdders(self, randomOrder=True, i2p=True, recent=0):
+        '''
+            Return a list of addresses
+        '''
+        conn = sqlite3.connect(self.addressDB, timeout=30)
+        c = conn.cursor()
+        if randomOrder:
+            addresses = c.execute('SELECT * FROM adders ORDER BY RANDOM();')
+        else:
+            addresses = c.execute('SELECT * FROM adders;')
+        addressList = []
+        for i in addresses:
+            if len(i[0].strip()) == 0:
+                continue
+            addressList.append(i[0])
+        conn.close()
+        testList = list(addressList) # create new list to iterate
+        for address in testList:
+            try:
+                if recent > 0 and (self._utils.getEpoch() - self.getAddressInfo(address, 'lastConnect')) > recent:
+                    raise TypeError # If there is no last-connected date or it was too long ago, don't add peer to list if recent is not 0
+            except TypeError:
+                addressList.remove(address)
+        return addressList
+
+    def listPeers(self, randomOrder=True, getPow=False, trust=0):
+        '''
+            Return a list of public keys (misleading function name)
+
+            randomOrder determines if the list should be in a random order
+            trust sets the minimum trust to list
+        '''
+        conn = sqlite3.connect(self.peerDB, timeout=30)
+        c = conn.cursor()
+
+        payload = ''
+
+        if trust not in (0, 1, 2):
+            logger.error('Tried to select invalid trust.')
+            return
+
+        if randomOrder:
+            payload = 'SELECT * FROM peers WHERE trust >= ? ORDER BY RANDOM();'
+        else:
+            payload = 'SELECT * FROM peers WHERE trust >= ?;'
+
+        peerList = []
+
+        for i in c.execute(payload, (trust,)):
+            try:
+                if len(i[0]) != 0:
+                    if getPow:
+                        peerList.append(i[0] + '-' + i[1])
+                    else:
+                        peerList.append(i[0])
+            except TypeError:
+                pass
+
+        conn.close()
+
+        return peerList
+
+    def getPeerInfo(self, peer, info):
+        '''
+            Get info about a peer from their database entry
+
+            id text             0
+            name text,          1
+            adders text,        2
+            dateSeen not null,  3
+            trust int           4
+            hashID text         5
+        '''
+        conn = sqlite3.connect(self.peerDB, timeout=30)
+        c = conn.cursor()
+
+        command = (peer,)
+        infoNumbers = {'id': 0, 'name': 1, 'adders': 2, 'dateSeen': 3, 'trust': 4, 'hashID': 5}
+        info = infoNumbers[info]
+        iterCount = 0
+        retVal = ''
+
+        for row in c.execute('SELECT * FROM peers WHERE id=?;', command):
+            for i in row:
+                if iterCount == info:
+                    retVal = i
+                    break
+                else:
+                    iterCount += 1
+
+        conn.close()
+
+        return retVal
+
+    def setPeerInfo(self, peer, key, data):
+        '''
+            Update a peer for a key
+        '''
+
+        conn = sqlite3.connect(self.peerDB, timeout=30)
+        c = conn.cursor()
+
+        command = (data, peer)
+
+        # TODO: validate key on whitelist
+        if key not in ('id', 'name', 'pubkey', 'forwardKey', 'dateSeen', 'trust'):
+            raise Exception("Got invalid database key when setting peer info")
+
+        c.execute('UPDATE peers SET ' + key + ' = ? WHERE id=?', command)
+        conn.commit()
+        conn.close()
+
+        return
+
+    def getAddressInfo(self, address, info):
+        '''
+            Get info about an address from its database entry
+
+            address text, 0
+            type int, 1
+            knownPeer text, 2
+            speed int, 3
+            success int, 4
+            powValue    5
+            failure int 6
+            lastConnect 7
+            trust       8
+            introduced  9
+        '''
+
+        conn = sqlite3.connect(self.addressDB, timeout=30)
+        c = conn.cursor()
+
+        command = (address,)
+        infoNumbers = {'address': 0, 'type': 1, 'knownPeer': 2, 'speed': 3, 'success': 4, 'powValue': 5, 'failure': 6, 'lastConnect': 7, 'trust': 8, 'introduced': 9}
+        info = infoNumbers[info]
+        iterCount = 0
+        retVal = ''
+
+        for row in c.execute('SELECT * FROM adders WHERE address=?;', command):
+            for i in row:
+                if iterCount == info:
+                    retVal = i
+                    break
+                else:
+                    iterCount += 1
+        conn.close()
+
+        return retVal
+
+    def setAddressInfo(self, address, key, data):
+        '''
+            Update an address for a key
+        '''
+
+        conn = sqlite3.connect(self.addressDB, timeout=30)
+        c = conn.cursor()
+
+        command = (data, address)
+
+        if key not in ('address', 'type', 'knownPeer', 'speed', 'success', 'failure', 'powValue', 'lastConnect', 'lastConnectAttempt', 'trust', 'introduced'):
+            raise Exception("Got invalid database key when setting address info")
+        else:
+            c.execute('UPDATE adders SET ' + key + ' = ? WHERE address=?', command)
+            conn.commit()
+        conn.close()
+
+        return
+
+    def getBlockList(self, dateRec = None, unsaved = False):
+        '''
+            Get list of our blocks
+        '''
+        if dateRec == None:
+            dateRec = 0
+
+        conn = sqlite3.connect(self.blockDB, timeout=30)
+        c = conn.cursor()
+
+        execute = 'SELECT hash FROM hashes WHERE dateReceived >= ? ORDER BY dateReceived ASC;'
+        args = (dateRec,)
+        rows = list()
+        for row in c.execute(execute, args):
+            for i in row:
+                rows.append(i)
+        conn.close()
+        return rows
+
+    def getBlockDate(self, blockHash):
+        '''
+            Returns the date a block was received
+        '''
+
+        conn = sqlite3.connect(self.blockDB, timeout=30)
+        c = conn.cursor()
+
+        execute = 'SELECT dateReceived FROM hashes WHERE hash=?;'
+        args = (blockHash,)
+        for row in c.execute(execute, args):
+            for i in row:
+                return int(i)
+        conn.close()
+        return None
+
+    def getBlocksByType(self, blockType, orderDate=True):
+        '''
+            Returns a list of blocks by the type
+        '''
+
+        conn = sqlite3.connect(self.blockDB, timeout=30)
+        c = conn.cursor()
+
+        if orderDate:
+            execute = 'SELECT hash FROM hashes WHERE dataType=? ORDER BY dateReceived;'
+        else:
+            execute = 'SELECT hash FROM hashes WHERE dataType=?;'
+
+        args = (blockType,)
+        rows = list()
+
+        for row in c.execute(execute, args):
+            for i in row:
+                rows.append(i)
+        conn.close()
+        return rows
+
+    def getExpiredBlocks(self):
+        '''Returns a list of expired blocks'''
+        conn = sqlite3.connect(self.blockDB, timeout=30)
+        c = conn.cursor()
+        date = int(self._utils.getEpoch())
+
+        execute = 'SELECT hash FROM hashes WHERE expire <= %s ORDER BY dateReceived;' % (date,)
+
+        rows = list()
+        for row in c.execute(execute):
+            for i in row:
+                rows.append(i)
+        conn.close()
+        return rows
+
+    def updateBlockInfo(self, hash, key, data):
+        '''
+            sets info associated with a block
+
+            hash         - the hash of a block
+            dateReceived - the date the block was recieved, not necessarily when it was created
+            decrypted    - if we can successfully decrypt the block (does not describe its current state)
+            dataType     - data type of the block
+            dataFound    - if the data has been found for the block
+            dataSaved    - if the data has been saved for the block
+            sig    - optional signature by the author (not optional if author is specified)
+            author       - multi-round partial sha3-256 hash of authors public key
+            dateClaimed  - timestamp claimed inside the block, only as trustworthy as the block author is
+            expire       - expire date for a block
+        '''
+
+        if key not in ('dateReceived', 'decrypted', 'dataType', 'dataFound', 'dataSaved', 'sig', 'author', 'dateClaimed', 'expire'):
+            return False
+
+        conn = sqlite3.connect(self.blockDB, timeout=30)
+        c = conn.cursor()
+        args = (data, hash)
+        c.execute("UPDATE hashes SET " + key + " = ? where hash = ?;", args)
+        conn.commit()
+        conn.close()
+
+        return True
+
+    def insertBlock(self, data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = {}, expire=None, disableForward=False):
+        '''
+            Inserts a block into the network
+            encryptType must be specified to encrypt a block
+        '''
+        allocationReachedMessage = 'Cannot insert block, disk allocation reached.'
+        if self._utils.storageCounter.isFull():
+            logger.error(allocationReachedMessage)
+            return False
+        retData = False
+
+        createTime = self._utils.getRoundedEpoch()
+
+        # check nonce
+        dataNonce = self._utils.bytesToStr(self._crypto.sha3Hash(data))
+        try:
+            with open(self.dataNonceFile, 'r') as nonces:
+                if dataNonce in nonces:
+                    return retData
+        except FileNotFoundError:
+            pass
+        # record nonce
+        with open(self.dataNonceFile, 'a') as nonceFile:
+            nonceFile.write(dataNonce + '\n')
+
+        if type(data) is bytes:
+            data = data.decode()
+        data = str(data)
+        plaintext = data
+        plaintextMeta = {}
+        plaintextPeer = asymPeer
+
+        retData = ''
+        signature = ''
+        signer = ''
+        metadata = {}
+        # metadata is full block metadata, meta is internal, user specified metadata
+
+        # only use header if not set in provided meta
+
+        meta['type'] = str(header)
+
+        if encryptType in ('asym', 'sym', ''):
+            metadata['encryptType'] = encryptType
+        else:
+            raise onionrexceptions.InvalidMetadata('encryptType must be asym or sym, or blank')
+
+        try:
+            data = data.encode()
+        except AttributeError:
+            pass
+
+        if encryptType == 'asym':
+            meta['rply'] = createTime # Duplicate the time in encrypted messages to prevent replays
+            if not disableForward and sign and asymPeer != self._crypto.pubKey:
+                try:
+                    forwardEncrypted = onionrusers.OnionrUser(self, asymPeer).forwardEncrypt(data)
+                    data = forwardEncrypted[0]
+                    meta['forwardEnc'] = True
+                    expire = forwardEncrypted[2] # Expire time of key. no sense keeping block after that
+                except onionrexceptions.InvalidPubkey:
+                    pass
+                    #onionrusers.OnionrUser(self, asymPeer).generateForwardKey()
+                fsKey = onionrusers.OnionrUser(self, asymPeer).generateForwardKey()
+                #fsKey = onionrusers.OnionrUser(self, asymPeer).getGeneratedForwardKeys().reverse()
+                meta['newFSKey'] = fsKey
+        jsonMeta = json.dumps(meta)
+        plaintextMeta = jsonMeta
+        if sign:
+            signature = self._crypto.edSign(jsonMeta.encode() + data, key=self._crypto.privKey, encodeResult=True)
+            signer = self._crypto.pubKey
+
+        if len(jsonMeta) > 1000:
+            raise onionrexceptions.InvalidMetadata('meta in json encoded form must not exceed 1000 bytes')
+
+        user = onionrusers.OnionrUser(self, symKey)
+
+        # encrypt block metadata/sig/content
+        if encryptType == 'sym':
+
+            if len(symKey) < self.requirements.passwordLength:
+                raise onionrexceptions.SecurityError('Weak encryption key')
+            jsonMeta = self._crypto.symmetricEncrypt(jsonMeta, key=symKey, returnEncoded=True).decode()
+            data = self._crypto.symmetricEncrypt(data, key=symKey, returnEncoded=True).decode()
+            signature = self._crypto.symmetricEncrypt(signature, key=symKey, returnEncoded=True).decode()
+            signer = self._crypto.symmetricEncrypt(signer, key=symKey, returnEncoded=True).decode()
+        elif encryptType == 'asym':
+            if self._utils.validatePubKey(asymPeer):
+                # Encrypt block data with forward secrecy key first, but not meta
+                jsonMeta = json.dumps(meta)
+                jsonMeta = self._crypto.pubKeyEncrypt(jsonMeta, asymPeer, encodedData=True).decode()
+                data = self._crypto.pubKeyEncrypt(data, asymPeer, encodedData=True).decode()
+                signature = self._crypto.pubKeyEncrypt(signature, asymPeer, encodedData=True).decode()
+                signer = self._crypto.pubKeyEncrypt(signer, asymPeer, encodedData=True).decode()
+                onionrusers.OnionrUser(self, asymPeer, saveUser=True)
+            else:
+                raise onionrexceptions.InvalidPubkey(asymPeer + ' is not a valid base32 encoded ed25519 key')
+
+        # compile metadata
+        metadata['meta'] = jsonMeta
+        metadata['sig'] = signature
+        metadata['signer'] = signer
+        metadata['time'] = createTime
+
+        # ensure expire is integer and of sane length
+        if type(expire) is not type(None):
+            assert len(str(int(expire))) < 14
+            metadata['expire'] = expire
+
+        # send block data (and metadata) to POW module to get tokenized block data
+        payload = subprocesspow.SubprocessPOW(data, metadata, self).start()
+        if payload != False:
+            try:
+                retData = self.setData(payload)
+            except onionrexceptions.DiskAllocationReached:
+                logger.error(allocationReachedMessage)
+                retData = False
+            else:
+                # Tell the api server through localCommand to wait for the daemon to upload this block to make stastical analysis more difficult
+                self._utils.localCommand('/waitforshare/' + retData, post=True)
+                self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
+                #self.setBlockType(retData, meta['type'])
+                self._utils.processBlockMetadata(retData)
+                self.daemonQueueAdd('uploadBlock', retData)
+
+        if retData != False:
+            if plaintextPeer == 'OVPCZLOXD6DC5JHX4EQ3PSOGAZ3T24F75HQLIUZSDSMYPEOXCPFA====':
+                events.event('insertdeniable', {'content': plaintext, 'meta': plaintextMeta, 'hash': retData, 'peer': self._utils.bytesToStr(asymPeer)}, onionr = self.onionrInst, threaded = True)
+            else:
+                events.event('insertblock', {'content': plaintext, 'meta': plaintextMeta, 'hash': retData, 'peer': self._utils.bytesToStr(asymPeer)}, onionr = self.onionrInst, threaded = True)
+        return retData
+
+    def introduceNode(self):
+        '''
+            Introduces our node into the network by telling X many nodes our HS address
+        '''
+
+        if(self._utils.isCommunicatorRunning(timeout=30)):
+            announceAmount = 2
+            nodeList = self.listAdders()
+
+            if len(nodeList) == 0:
+                for i in self.bootstrapList:
+                    if self._utils.validateID(i):
+                        self.addAddress(i)
+                        nodeList.append(i)
+
+            if announceAmount > len(nodeList):
+                announceAmount = len(nodeList)
+
+            for i in range(announceAmount):
+                self.daemonQueueAdd('announceNode', nodeList[i])
+
+            events.event('introduction', onionr = None)
+
+            return True
+        else:
+            logger.error('Onionr daemon is not running.')
+            return False
+        return

onionr/dbcreator.py

@@ -0,0 +1,156 @@
+'''
+    Onionr - P2P Anonymous Data Storage & Sharing
+
+    DBCreator, creates sqlite3 databases used by Onionr
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sqlite3, os
+class DBCreator:
+    def __init__(self, coreInst):
+        self.core = coreInst
+
+    def createAddressDB(self):
+        '''
+            Generate the address database
+
+            types:
+                1: I2P b32 address
+                2: Tor v2 (like facebookcorewwwi.onion)
+                3: Tor v3
+        '''
+        conn = sqlite3.connect(self.core.addressDB)
+        c = conn.cursor()
+        c.execute('''CREATE TABLE adders(
+            address text,
+            type int,
+            knownPeer text,
+            speed int,
+            success int,
+            powValue text,
+            failure int,
+            lastConnect int,
+            lastConnectAttempt int,
+            trust int,
+            introduced int
+            );
+        ''')
+        conn.commit()
+        conn.close()
+
+    def createPeerDB(self):
+        '''
+            Generate the peer sqlite3 database and populate it with the peers table.
+        '''
+        # generate the peer database
+        conn = sqlite3.connect(self.core.peerDB)
+        c = conn.cursor()
+        c.execute('''CREATE TABLE peers(
+            ID text not null,
+            name text,
+            adders text,
+            dateSeen not null,
+            trust int,
+            hashID text);
+        ''')
+        c.execute('''CREATE TABLE forwardKeys(
+        peerKey text not null,
+        forwardKey text not null,
+        date int not null,
+        expire int not null
+        );''')
+        conn.commit()
+        conn.close()
+        return
+
+    def createBlockDB(self):
+        '''
+            Create a database for blocks
+
+            hash         - the hash of a block
+            dateReceived - the date the block was recieved, not necessarily when it was created
+            decrypted    - if we can successfully decrypt the block (does not describe its current state)
+            dataType     - data type of the block
+            dataFound    - if the data has been found for the block
+            dataSaved    - if the data has been saved for the block
+            sig    - optional signature by the author (not optional if author is specified)
+            author       - multi-round partial sha3-256 hash of authors public key
+            dateClaimed  - timestamp claimed inside the block, only as trustworthy as the block author is
+            expire int   - block expire date in epoch
+        '''
+        if os.path.exists(self.core.blockDB):
+            raise FileExistsError("Block database already exists")
+        conn = sqlite3.connect(self.core.blockDB)
+        c = conn.cursor()
+        c.execute('''CREATE TABLE hashes(
+            hash text not null,
+            dateReceived int,
+            decrypted int,
+            dataType text,
+            dataFound int,
+            dataSaved int,
+            sig text,
+            author text,
+            dateClaimed int,
+            expire int
+            );
+        ''')
+        conn.commit()
+        conn.close()
+        return
+    
+    def createBlockDataDB(self):
+        if os.path.exists(self.core.blockDataDB):
+            raise FileExistsError("Block data database already exists")
+        conn = sqlite3.connect(self.core.blockDataDB)
+        c = conn.cursor()
+        c.execute('''CREATE TABLE blockData(
+            hash text not null,
+            data blob not null
+            );
+        ''')
+        conn.commit()
+        conn.close()
+
+    def createForwardKeyDB(self):
+        '''
+            Create the forward secrecy key db (*for *OUR* keys*)
+        '''
+        if os.path.exists(self.core.forwardKeysFile):
+            raise FileExistsError("Block database already exists")
+        conn = sqlite3.connect(self.core.forwardKeysFile)
+        c = conn.cursor()
+        c.execute('''CREATE TABLE myForwardKeys(
+            peer text not null,
+            publickey text not null,
+            privatekey text not null,
+            date int not null,
+            expire int not null
+            );
+        ''')
+        conn.commit()
+        conn.close()
+        return
+    
+    def createDaemonDB(self):
+        '''
+            Create the daemon queue database
+        '''
+        conn = sqlite3.connect(self.core.queueDB, timeout=10)
+        c = conn.cursor()
+        # Create table
+        c.execute('''CREATE TABLE commands (id integer primary key autoincrement, command text, data text, date text, responseID text)''')
+        conn.commit()
+        conn.close()

onionr/dependencies/secrets.py

@@ -0,0 +1,331 @@
+"""Generate cryptographically strong pseudo-random numbers suitable for
+managing secrets such as account authentication, tokens, and similar.
+
+See PEP 506 for more information.
+https://www.python.org/dev/peps/pep-0506/
+
+
+A. HISTORY OF THE SOFTWARE
+==========================
+
+Python was created in the early 1990s by Guido van Rossum at Stichting
+Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands
+as a successor of a language called ABC.  Guido remains Python's
+principal author, although it includes many contributions from others.
+
+In 1995, Guido continued his work on Python at the Corporation for
+National Research Initiatives (CNRI, see http://www.cnri.reston.va.us)
+in Reston, Virginia where he released several versions of the
+software.
+
+In May 2000, Guido and the Python core development team moved to
+BeOpen.com to form the BeOpen PythonLabs team.  In October of the same
+year, the PythonLabs team moved to Digital Creations, which became
+Zope Corporation.  In 2001, the Python Software Foundation (PSF, see
+https://www.python.org/psf/) was formed, a non-profit organization
+created specifically to own Python-related Intellectual Property.
+Zope Corporation was a sponsoring member of the PSF.
+
+All Python releases are Open Source (see http://www.opensource.org for
+the Open Source Definition).  Historically, most, but not all, Python
+releases have also been GPL-compatible; the table below summarizes
+the various releases.
+
+    Release         Derived     Year        Owner       GPL-
+                    from                                compatible? (1)
+
+    0.9.0 thru 1.2              1991-1995   CWI         yes
+    1.3 thru 1.5.2  1.2         1995-1999   CNRI        yes
+    1.6             1.5.2       2000        CNRI        no
+    2.0             1.6         2000        BeOpen.com  no
+    1.6.1           1.6         2001        CNRI        yes (2)
+    2.1             2.0+1.6.1   2001        PSF         no
+    2.0.1           2.0+1.6.1   2001        PSF         yes
+    2.1.1           2.1+2.0.1   2001        PSF         yes
+    2.1.2           2.1.1       2002        PSF         yes
+    2.1.3           2.1.2       2002        PSF         yes
+    2.2 and above   2.1.1       2001-now    PSF         yes
+
+Footnotes:
+
+(1) GPL-compatible doesn't mean that we're distributing Python under
+    the GPL.  All Python licenses, unlike the GPL, let you distribute
+    a modified version without making your changes open source.  The
+    GPL-compatible licenses make it possible to combine Python with
+    other software that is released under the GPL; the others don't.
+
+(2) According to Richard Stallman, 1.6.1 is not GPL-compatible,
+    because its license has a choice of law clause.  According to
+    CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1
+    is "not incompatible" with the GPL.
+
+Thanks to the many outside volunteers who have worked under Guido's
+direction to make these releases possible.
+
+
+B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON
+===============================================================
+
+PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+--------------------------------------------
+
+1. This LICENSE AGREEMENT is between the Python Software Foundation
+("PSF"), and the Individual or Organization ("Licensee") accessing and
+otherwise using this software ("Python") in source or binary form and
+its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, PSF hereby
+grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
+analyze, test, perform and/or display publicly, prepare derivative works,
+distribute, and otherwise use Python alone or in any derivative version,
+provided, however, that PSF's License Agreement and PSF's notice of copyright,
+i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
+2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Python Software Foundation; All
+Rights Reserved" are retained in Python alone or in any derivative version
+prepared by Licensee.
+
+3. In the event Licensee prepares a derivative work that is based on
+or incorporates Python or any part thereof, and wants to make
+the derivative work available to others as provided herein, then
+Licensee hereby agrees to include in any such work a brief summary of
+the changes made to Python.
+
+4. PSF is making Python available to Licensee on an "AS IS"
+basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
+A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
+OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+7. Nothing in this License Agreement shall be deemed to create any
+relationship of agency, partnership, or joint venture between PSF and
+Licensee.  This License Agreement does not grant permission to use PSF
+trademarks or trade name in a trademark sense to endorse or promote
+products or services of Licensee, or any third party.
+
+8. By copying, installing or otherwise using Python, Licensee
+agrees to be bound by the terms and conditions of this License
+Agreement.
+
+
+BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0
+-------------------------------------------
+
+BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1
+
+1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an
+office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the
+Individual or Organization ("Licensee") accessing and otherwise using
+this software in source or binary form and its associated
+documentation ("the Software").
+
+2. Subject to the terms and conditions of this BeOpen Python License
+Agreement, BeOpen hereby grants Licensee a non-exclusive,
+royalty-free, world-wide license to reproduce, analyze, test, perform
+and/or display publicly, prepare derivative works, distribute, and
+otherwise use the Software alone or in any derivative version,
+provided, however, that the BeOpen Python License is retained in the
+Software, alone or in any derivative version prepared by Licensee.
+
+3. BeOpen is making the Software available to Licensee on an "AS IS"
+basis.  BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE
+SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS
+AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY
+DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+5. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+6. This License Agreement shall be governed by and interpreted in all
+respects by the law of the State of California, excluding conflict of
+law provisions.  Nothing in this License Agreement shall be deemed to
+create any relationship of agency, partnership, or joint venture
+between BeOpen and Licensee.  This License Agreement does not grant
+permission to use BeOpen trademarks or trade names in a trademark
+sense to endorse or promote products or services of Licensee, or any
+third party.  As an exception, the "BeOpen Python" logos available at
+http://www.pythonlabs.com/logos.html may be used according to the
+permissions granted on that web page.
+
+7. By copying, installing or otherwise using the software, Licensee
+agrees to be bound by the terms and conditions of this License
+Agreement.
+
+
+CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1
+---------------------------------------
+
+1. This LICENSE AGREEMENT is between the Corporation for National
+Research Initiatives, having an office at 1895 Preston White Drive,
+Reston, VA 20191 ("CNRI"), and the Individual or Organization
+("Licensee") accessing and otherwise using Python 1.6.1 software in
+source or binary form and its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, CNRI
+hereby grants Licensee a nonexclusive, royalty-free, world-wide
+license to reproduce, analyze, test, perform and/or display publicly,
+prepare derivative works, distribute, and otherwise use Python 1.6.1
+alone or in any derivative version, provided, however, that CNRI's
+License Agreement and CNRI's notice of copyright, i.e., "Copyright (c)
+1995-2001 Corporation for National Research Initiatives; All Rights
+Reserved" are retained in Python 1.6.1 alone or in any derivative
+version prepared by Licensee.  Alternately, in lieu of CNRI's License
+Agreement, Licensee may substitute the following text (omitting the
+quotes): "Python 1.6.1 is made available subject to the terms and
+conditions in CNRI's License Agreement.  This Agreement together with
+Python 1.6.1 may be located on the Internet using the following
+unique, persistent identifier (known as a handle): 1895.22/1013.  This
+Agreement may also be obtained from a proxy server on the Internet
+using the following URL: http://hdl.handle.net/1895.22/1013".
+
+3. In the event Licensee prepares a derivative work that is based on
+or incorporates Python 1.6.1 or any part thereof, and wants to make
+the derivative work available to others as provided herein, then
+Licensee hereby agrees to include in any such work a brief summary of
+the changes made to Python 1.6.1.
+
+4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS"
+basis.  CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
+A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1,
+OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+7. This License Agreement shall be governed by the federal
+intellectual property law of the United States, including without
+limitation the federal copyright law, and, to the extent such
+U.S. federal law does not apply, by the law of the Commonwealth of
+Virginia, excluding Virginia's conflict of law provisions.
+Notwithstanding the foregoing, with regard to derivative works based
+on Python 1.6.1 that incorporate non-separable material that was
+previously distributed under the GNU General Public License (GPL), the
+law of the Commonwealth of Virginia shall govern this License
+Agreement only as to issues arising under or with respect to
+Paragraphs 4, 5, and 7 of this License Agreement.  Nothing in this
+License Agreement shall be deemed to create any relationship of
+agency, partnership, or joint venture between CNRI and Licensee.  This
+License Agreement does not grant permission to use CNRI trademarks or
+trade name in a trademark sense to endorse or promote products or
+services of Licensee, or any third party.
+
+8. By clicking on the "ACCEPT" button where indicated, or by copying,
+installing or otherwise using Python 1.6.1, Licensee agrees to be
+bound by the terms and conditions of this License Agreement.
+
+        ACCEPT
+
+
+CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2
+--------------------------------------------------
+
+Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam,
+The Netherlands.  All rights reserved.
+
+Permission to use, copy, modify, and distribute this software and its
+documentation for any purpose and without fee is hereby granted,
+provided that the above copyright notice appear in all copies and that
+both that copyright notice and this permission notice appear in
+supporting documentation, and that the name of Stichting Mathematisch
+Centrum or CWI not be used in advertising or publicity pertaining to
+distribution of the software without specific, written prior
+permission.
+
+STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO
+THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE
+FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
+OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+
+"""
+
+__all__ = ['choice', 'randbelow', 'randbits', 'SystemRandom',
+           'token_bytes', 'token_hex', 'token_urlsafe',
+           'compare_digest',
+           ]
+
+
+import base64
+import binascii
+import os
+
+from hmac import compare_digest
+from random import SystemRandom
+
+_sysrand = SystemRandom()
+
+randbits = _sysrand.getrandbits
+choice = _sysrand.choice
+
+def randbelow(exclusive_upper_bound):
+    """Return a random int in the range [0, n)."""
+    if exclusive_upper_bound <= 0:
+        raise ValueError("Upper bound must be positive.")
+    return _sysrand._randbelow(exclusive_upper_bound)
+
+DEFAULT_ENTROPY = 32  # number of bytes to return by default
+
+def token_bytes(nbytes=None):
+    """Return a random byte string containing *nbytes* bytes.
+
+    If *nbytes* is ``None`` or not supplied, a reasonable
+    default is used.
+
+    >>> token_bytes(16)  #doctest:+SKIP
+    b'\\xebr\\x17D*t\\xae\\xd4\\xe3S\\xb6\\xe2\\xebP1\\x8b'
+
+    """
+    if nbytes is None:
+        nbytes = DEFAULT_ENTROPY
+    return os.urandom(nbytes)
+
+def token_hex(nbytes=None):
+    """Return a random text string, in hexadecimal.
+
+    The string has *nbytes* random bytes, each byte converted to two
+    hex digits.  If *nbytes* is ``None`` or not supplied, a reasonable
+    default is used.
+
+    >>> token_hex(16)  #doctest:+SKIP
+    'f9bf78b9a18ce6d46a0cd2b0b86df9da'
+
+    """
+    return binascii.hexlify(token_bytes(nbytes)).decode('ascii')
+
+def token_urlsafe(nbytes=None):
+    """Return a random URL-safe text string, in Base64 encoding.
+
+    The string has *nbytes* random bytes.  If *nbytes* is ``None``
+    or not supplied, a reasonable default is used.
+
+    >>> token_urlsafe(16)  #doctest:+SKIP
+    'Drmhze6EPcv0fN_81Bj-nA'
+
+    """
+    tok = token_bytes(nbytes)
+    return base64.urlsafe_b64encode(tok).rstrip(b'=').decode('ascii')
+

onionr/etc/onionrvalues.py

@@ -0,0 +1,24 @@
+'''
+    Onionr - P2P Microblogging Platform & Social network
+
+    This file defines values and requirements used by Onionr
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+class OnionrValues:
+    def __init__(self):
+        self.passwordLength = 20
+        self.blockMetadataLengths = {'meta': 1000, 'sig': 200, 'signer': 200, 'time': 10, 'powRandomToken': 1000, 'encryptType': 4, 'expire': 14} #TODO properly refine values to minimum needed

onionr/etc/pgpwords.py

@@ -0,0 +1,297 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*- (because 0xFF, even : "Yucatán")
+
+'''This file is adapted from https://github.com/thblt/pgp-words by github user 'thblt' ('Thibault Polge), GPL v3 license'''
+
+import os, re, sys, binascii
+
+_words = [
+    ["aardvark", "adroitness"],
+    ["absurd", "adviser"],
+    ["accrue", "aftermath"],
+    ["acme", "aggregate"],
+    ["adrift", "alkali"],
+    ["adult", "almighty"],
+    ["afflict", "amulet"],
+    ["ahead", "amusement"],
+    ["aimless", "antenna"],
+    ["Algol", "applicant"],
+    ["allow", "Apollo"],
+    ["alone", "armistice"],
+    ["ammo", "article"],
+    ["ancient", "asteroid"],
+    ["apple", "Atlantic"],
+    ["artist", "atmosphere"],
+    ["assume", "autopsy"],
+    ["Athens", "Babylon"],
+    ["atlas", "backwater"],
+    ["Aztec", "barbecue"],
+    ["baboon", "belowground"],
+    ["backfield", "bifocals"],
+    ["backward", "bodyguard"],
+    ["banjo", "bookseller"],
+    ["beaming", "borderline"],
+    ["bedlamp", "bottomless"],
+    ["beehive", "Bradbury"],
+    ["beeswax", "bravado"],
+    ["befriend", "Brazilian"],
+    ["Belfast", "breakaway"],
+    ["berserk", "Burlington"],
+    ["billiard", "businessman"],
+    ["bison", "butterfat"],
+    ["blackjack", "Camelot"],
+    ["blockade", "candidate"],
+    ["blowtorch", "cannonball"],
+    ["bluebird", "Capricorn"],
+    ["bombast", "caravan"],
+    ["bookshelf", "caretaker"],
+    ["brackish", "celebrate"],
+    ["breadline", "cellulose"],
+    ["breakup", "certify"],
+    ["brickyard", "chambermaid"],
+    ["briefcase", "Cherokee"],
+    ["Burbank", "Chicago"],
+    ["button", "clergyman"],
+    ["buzzard", "coherence"],
+    ["cement", "combustion"],
+    ["chairlift", "commando"],
+    ["chatter", "company"],
+    ["checkup", "component"],
+    ["chisel", "concurrent"],
+    ["choking", "confidence"],
+    ["chopper", "conformist"],
+    ["Christmas", "congregate"],
+    ["clamshell", "consensus"],
+    ["classic", "consulting"],
+    ["classroom", "corporate"],
+    ["cleanup", "corrosion"],
+    ["clockwork", "councilman"],
+    ["cobra", "crossover"],
+    ["commence", "crucifix"],
+    ["concert", "cumbersome"],
+    ["cowbell", "customer"],
+    ["crackdown", "Dakota"],
+    ["cranky", "decadence"],
+    ["crowfoot", "December"],
+    ["crucial", "decimal"],
+    ["crumpled", "designing"],
+    ["crusade", "detector"],
+    ["cubic", "detergent"],
+    ["dashboard", "determine"],
+    ["deadbolt", "dictator"],
+    ["deckhand", "dinosaur"],
+    ["dogsled", "direction"],
+    ["dragnet", "disable"],
+    ["drainage", "disbelief"],
+    ["dreadful", "disruptive"],
+    ["drifter", "distortion"],
+    ["dropper", "document"],
+    ["drumbeat", "embezzle"],
+    ["drunken", "enchanting"],
+    ["Dupont", "enrollment"],
+    ["dwelling", "enterprise"],
+    ["eating", "equation"],
+    ["edict", "equipment"],
+    ["egghead", "escapade"],
+    ["eightball", "Eskimo"],
+    ["endorse", "everyday"],
+    ["endow", "examine"],
+    ["enlist", "existence"],
+    ["erase", "exodus"],
+    ["escape", "fascinate"],
+    ["exceed", "filament"],
+    ["eyeglass", "finicky"],
+    ["eyetooth", "forever"],
+    ["facial", "fortitude"],
+    ["fallout", "frequency"],
+    ["flagpole", "gadgetry"],
+    ["flatfoot", "Galveston"],
+    ["flytrap", "getaway"],
+    ["fracture", "glossary"],
+    ["framework", "gossamer"],
+    ["freedom", "graduate"],
+    ["frighten", "gravity"],
+    ["gazelle", "guitarist"],
+    ["Geiger", "hamburger"],
+    ["glitter", "Hamilton"],
+    ["glucose", "handiwork"],
+    ["goggles", "hazardous"],
+    ["goldfish", "headwaters"],
+    ["gremlin", "hemisphere"],
+    ["guidance", "hesitate"],
+    ["hamlet", "hideaway"],
+    ["highchair", "holiness"],
+    ["hockey", "hurricane"],
+    ["indoors", "hydraulic"],
+    ["indulge", "impartial"],
+    ["inverse", "impetus"],
+    ["involve", "inception"],
+    ["island", "indigo"],
+    ["jawbone", "inertia"],
+    ["keyboard", "infancy"],
+    ["kickoff", "inferno"],
+    ["kiwi", "informant"],
+    ["klaxon", "insincere"],
+    ["locale", "insurgent"],
+    ["lockup", "integrate"],
+    ["merit", "intention"],
+    ["minnow", "inventive"],
+    ["miser", "Istanbul"],
+    ["Mohawk", "Jamaica"],
+    ["mural", "Jupiter"],
+    ["music", "leprosy"],
+    ["necklace", "letterhead"],
+    ["Neptune", "liberty"],
+    ["newborn", "maritime"],
+    ["nightbird", "matchmaker"],
+    ["Oakland", "maverick"],
+    ["obtuse", "Medusa"],
+    ["offload", "megaton"],
+    ["optic", "microscope"],
+    ["orca", "microwave"],
+    ["payday", "midsummer"],
+    ["peachy", "millionaire"],
+    ["pheasant", "miracle"],
+    ["physique", "misnomer"],
+    ["playhouse", "molasses"],
+    ["Pluto", "molecule"],
+    ["preclude", "Montana"],
+    ["prefer", "monument"],
+    ["preshrunk", "mosquito"],
+    ["printer", "narrative"],
+    ["prowler", "nebula"],
+    ["pupil", "newsletter"],
+    ["puppy", "Norwegian"],
+    ["python", "October"],
+    ["quadrant", "Ohio"],
+    ["quiver", "onlooker"],
+    ["quota", "opulent"],
+    ["ragtime", "Orlando"],
+    ["ratchet", "outfielder"],
+    ["rebirth", "Pacific"],
+    ["reform", "pandemic"],
+    ["regain", "Pandora"],
+    ["reindeer", "paperweight"],
+    ["rematch", "paragon"],
+    ["repay", "paragraph"],
+    ["retouch", "paramount"],
+    ["revenge", "passenger"],
+    ["reward", "pedigree"],
+    ["rhythm", "Pegasus"],
+    ["ribcage", "penetrate"],
+    ["ringbolt", "perceptive"],
+    ["robust", "performance"],
+    ["rocker", "pharmacy"],
+    ["ruffled", "phonetic"],
+    ["sailboat", "photograph"],
+    ["sawdust", "pioneer"],
+    ["scallion", "pocketful"],
+    ["scenic", "politeness"],
+    ["scorecard", "positive"],
+    ["Scotland", "potato"],
+    ["seabird", "processor"],
+    ["select", "provincial"],
+    ["sentence", "proximate"],
+    ["shadow", "puberty"],
+    ["shamrock", "publisher"],
+    ["showgirl", "pyramid"],
+    ["skullcap", "quantity"],
+    ["skydive", "racketeer"],
+    ["slingshot", "rebellion"],
+    ["slowdown", "recipe"],
+    ["snapline", "recover"],
+    ["snapshot", "repellent"],
+    ["snowcap", "replica"],
+    ["snowslide", "reproduce"],
+    ["solo", "resistor"],
+    ["southward", "responsive"],
+    ["soybean", "retraction"],
+    ["spaniel", "retrieval"],
+    ["spearhead", "retrospect"],
+    ["spellbind", "revenue"],
+    ["spheroid", "revival"],
+    ["spigot", "revolver"],
+    ["spindle", "sandalwood"],
+    ["spyglass", "sardonic"],
+    ["stagehand", "Saturday"],
+    ["stagnate", "savagery"],
+    ["stairway", "scavenger"],
+    ["standard", "sensation"],
+    ["stapler", "sociable"],
+    ["steamship", "souvenir"],
+    ["sterling", "specialist"],
+    ["stockman", "speculate"],
+    ["stopwatch", "stethoscope"],
+    ["stormy", "stupendous"],
+    ["sugar", "supportive"],
+    ["surmount", "surrender"],
+    ["suspense", "suspicious"],
+    ["sweatband", "sympathy"],
+    ["swelter", "tambourine"],
+    ["tactics", "telephone"],
+    ["talon", "therapist"],
+    ["tapeworm", "tobacco"],
+    ["tempest", "tolerance"],
+    ["tiger", "tomorrow"],
+    ["tissue", "torpedo"],
+    ["tonic", "tradition"],
+    ["topmost", "travesty"],
+    ["tracker", "trombonist"],
+    ["transit", "truncated"],
+    ["trauma", "typewriter"],
+    ["treadmill", "ultimate"],
+    ["Trojan", "undaunted"],
+    ["trouble", "underfoot"],
+    ["tumor", "unicorn"],
+    ["tunnel", "unify"],
+    ["tycoon", "universe"],
+    ["uncut", "unravel"],
+    ["unearth", "upcoming"],
+    ["unwind", "vacancy"],
+    ["uproot", "vagabond"],
+    ["upset", "vertigo"],
+    ["upshot", "Virginia"],
+    ["vapor", "visitor"],
+    ["village", "vocalist"],
+    ["virus", "voyager"],
+    ["Vulcan", "warranty"],
+    ["waffle", "Waterloo"],
+    ["wallet", "whimsical"],
+    ["watchword", "Wichita"],
+    ["wayside", "Wilmington"],
+    ["willow", "Wyoming"],
+    ["woodlark", "yesteryear"],
+    ["Zulu", "Yucatan"]]
+
+hexre = re.compile("[a-fA-F0-9]+")
+
+def wordify(seq):
+    seq = filter(lambda x: x not in (' ', '\n', '\t'), seq)
+    seq = "".join(seq) # Python3 compatibility
+
+    if not hexre.match(seq):
+        raise Exception("Input is not a valid hexadecimal value.")
+
+    if len(seq) % 2:
+        raise Exception("Input contains an odd number of bytes.")
+
+    ret = []
+    for i in range(0, len(seq), 2):
+        ret.append(_words[int(seq[i:i+2], 16)][(i//2)%2].lower())
+    return ret
+
+def hexify(seq, delim=' '):
+    ret = b''
+    sentence = seq
+    try:
+        sentence = seq.split(delim)
+    except AttributeError:
+        pass
+    count = 0
+    for word in sentence:
+        count = 0
+        for wordPair in _words:
+            if word in wordPair:
+                ret += bytes([(count)])
+            count += 1
+    return binascii.hexlify(ret)

onionr/httpapi/__init__.py

@@ -0,0 +1,29 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This file registers plugin's flask blueprints for the client http server
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import onionrplugins
+
+def load_plugin_blueprints(flaskapp):
+    '''Iterate enabled plugins and load any http endpoints they have'''
+    for plugin in onionrplugins.get_enabled_plugins():
+        plugin = onionrplugins.get_plugin(plugin)
+        try:
+            flaskapp.register_blueprint(getattr(plugin, 'flask_blueprint'))
+        except AttributeError:
+            pass

onionr/httpapi/friendsapi/__init__.py

@@ -0,0 +1,56 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This file creates http endpoints for friend management
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import core, json
+from onionrusers import contactmanager
+from flask import Blueprint, Response, request, abort, redirect
+
+friends = Blueprint('friends', __name__)
+
+@friends.route('/friends/list')
+def list_friends():
+    pubkey_list = {}
+    friend_list = contactmanager.ContactManager.list_friends(core.Core())
+    for friend in friend_list:
+        pubkey_list[friend.publicKey] = {'name': friend.get_info('name')}
+    return json.dumps(pubkey_list)
+
+@friends.route('/friends/add/<pubkey>', methods=['POST'])
+def add_friend(pubkey):
+    contactmanager.ContactManager(core.Core(), pubkey, saveUser=True).setTrust(1)
+    return redirect(request.referrer + '#' + request.form['token'])
+
+@friends.route('/friends/remove/<pubkey>', methods=['POST'])
+def remove_friend(pubkey):
+    contactmanager.ContactManager(core.Core(), pubkey).setTrust(0)
+    return redirect(request.referrer + '#' + request.form['token'])
+
+@friends.route('/friends/setinfo/<pubkey>/<key>', methods=['POST'])
+def set_info(pubkey, key):
+    data = request.form['data']
+    contactmanager.ContactManager(core.Core(), pubkey).set_info(key, data)
+    return redirect(request.referrer + '#' + request.form['token'])
+
+@friends.route('/friends/getinfo/<pubkey>/<key>')
+def get_info(pubkey, key):
+    retData = contactmanager.ContactManager(core.Core(), pubkey).get_info(key)
+    if retData is None:
+        abort(404)
+    else:
+        return retData

onionr/httpapi/simplecache/__init__.py

@@ -0,0 +1,31 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This file creates http endpoints for friend management
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import core
+from flask import Blueprint, Response, request, abort
+
+simplecache = Blueprint('simplecache', __name__)
+
+@simplecache.route('/get/<key>')
+def get_key(key):
+    return
+
+@simplecache.route('/set/<key>', methods=['POST'])
+def set_key(key):
+    return

onionr/keymanager.py

@@ -0,0 +1,80 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    Load, save, and delete the user's public key pairs (does not handle peer keys)
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import onionrcrypto
+class KeyManager:
+    def __init__(self, crypto):
+        assert isinstance(crypto, onionrcrypto.OnionrCrypto)
+        self._core = crypto._core
+        self._utils = self._core._utils
+        self.keyFile = crypto._keyFile
+        self.crypto = crypto
+
+    def addKey(self, pubKey=None, privKey=None):
+        if type(pubKey) is type(None) and type(privKey) is type(None):
+            pubKey, privKey = self.crypto.generatePubKey()
+        pubKey = self.crypto._core._utils.bytesToStr(pubKey)
+        privKey = self.crypto._core._utils.bytesToStr(privKey)
+        try:
+            if pubKey in self.getPubkeyList():
+                raise ValueError('Pubkey already in list: %s' % (pubKey,))
+        except FileNotFoundError:
+            pass
+        with open(self.keyFile, "a") as keyFile:
+            keyFile.write(pubKey + ',' + privKey + '\n')
+        return (pubKey, privKey)
+
+    def removeKey(self, pubKey):
+        '''Remove a key pair by pubkey'''
+        keyList = self.getPubkeyList()
+        keyData = ''
+        try:
+            keyList.remove(pubKey)
+        except ValueError:
+            return False
+        else:
+            keyData = ','.join(keyList)
+            with open(self.keyFile, "w") as keyFile:
+                keyFile.write(keyData)
+
+    def getPubkeyList(self):
+        '''Return a list of the user's keys'''
+        keyList = []
+        with open(self.keyFile, "r") as keyFile:
+            keyData = keyFile.read()
+        keyData = keyData.split('\n')
+        for pair in keyData:
+            if len(pair) > 0: keyList.append(pair.split(',')[0])
+        return keyList
+    
+    def getPrivkey(self, pubKey):
+        privKey = None
+        with open(self.keyFile, "r") as keyFile:
+            keyData = keyFile.read()
+        for pair in keyData.split('\n'):
+            if pubKey in pair:
+                privKey = pair.split(',')[1]
+        return privKey
+    
+    def changeActiveKey(self, pubKey):
+        '''Change crypto.pubKey and crypto.privKey to a given key pair by specifying the public key'''
+        if not pubKey in self.getPubkeyList():
+            raise ValueError('That pubkey does not exist')
+        self.crypto.pubKey = pubKey
+        self.crypto.privKey = self.getPrivkey(pubKey)

onionr/logger.py

@@ -0,0 +1,250 @@
+'''
+    Onionr - P2P Microblogging Platform & Social network
+
+    This file handles all operations involving logging
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+import re, sys, time, traceback, os
+
+class colors:
+    '''
+        This class allows you to set the color if ANSI codes are supported
+    '''
+    reset='\033[0m'
+    bold='\033[01m'
+    disable='\033[02m'
+    underline='\033[04m'
+    reverse='\033[07m'
+    strikethrough='\033[09m'
+    invisible='\033[08m'
+    italics='\033[3m'
+    class fg:
+        black='\033[30m'
+        red='\033[31m'
+        green='\033[32m'
+        orange='\033[33m'
+        blue='\033[34m'
+        purple='\033[35m'
+        cyan='\033[36m'
+        lightgrey='\033[37m'
+        darkgrey='\033[90m'
+        lightred='\033[91m'
+        lightgreen='\033[92m'
+        yellow='\033[93m'
+        lightblue='\033[94m'
+        pink='\033[95m'
+        lightcyan='\033[96m'
+    class bg:
+        black='\033[40m'
+        red='\033[41m'
+        green='\033[42m'
+        orange='\033[43m'
+        blue='\033[44m'
+        purple='\033[45m'
+        cyan='\033[46m'
+        lightgrey='\033[47m'
+    @staticmethod
+    def filter(data):
+        return re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]').sub('', str(data))
+
+'''
+    Use the bitwise operators to merge these settings
+'''
+USE_ANSI = 0b100
+if os.name == 'nt':
+    USE_ANSI = 0b000
+OUTPUT_TO_CONSOLE = 0b010
+OUTPUT_TO_FILE = 0b001
+
+LEVEL_DEBUG = 1
+LEVEL_INFO = 2
+LEVEL_WARN = 3
+LEVEL_ERROR = 4
+LEVEL_FATAL = 5
+LEVEL_IMPORTANT = 6
+
+_type = OUTPUT_TO_CONSOLE | USE_ANSI # the default settings for logging
+_level = LEVEL_DEBUG # the lowest level to log
+_outputfile = 'data/onionr.log' # the file to log to
+
+def set_settings(type):
+    '''
+        Set the settings for the logger using bitwise operators
+    '''
+
+    global _type
+    _type = type
+
+def get_settings():
+    '''
+        Get settings from the logger
+    '''
+
+    return _type
+
+def set_level(level):
+    '''
+        Set the lowest log level to output
+    '''
+
+    global _level
+    _level = level
+
+def get_level():
+    '''
+        Get the lowest log level currently being outputted
+    '''
+
+    return _level
+
+def set_file(outputfile):
+    '''
+        Set the file to output to, if enabled
+    '''
+
+    global _outputfile
+    _outputfile = outputfile
+
+def get_file():
+    '''
+        Get the file to output to
+    '''
+
+    return _outputfile
+
+def raw(data, fd = sys.stdout, sensitive = False):
+    '''
+        Outputs raw data to console without formatting
+    '''
+
+    if get_settings() & OUTPUT_TO_CONSOLE:
+        ts = fd.write('%s\n' % data)
+    if get_settings() & OUTPUT_TO_FILE and not sensitive:
+        try:
+            with open(_outputfile, "a+") as f:
+                f.write(colors.filter(data) + '\n')
+        except OSError:
+            pass
+
+def log(prefix, data, color = '', timestamp=True, fd = sys.stdout, prompt = True, sensitive = False):
+    '''
+        Logs the data
+        prefix : The prefix to the output
+        data   : The actual data to output
+        color  : The color to output before the data
+    '''
+    curTime = ''
+    if timestamp:
+        curTime = time.strftime("%m-%d %H:%M:%S") + ' '
+
+    output = colors.reset + str(color) + ('[' + colors.bold + str(prefix) + colors.reset + str(color) + '] ' if prompt is True else '') + curTime + str(data) + colors.reset
+    if not get_settings() & USE_ANSI:
+        output = colors.filter(output)
+
+    raw(output, fd = fd, sensitive = sensitive)
+
+def readline(message = ''):
+    '''
+        Takes in input from the console, not stored in logs
+        message: The message to display before taking input
+    '''
+
+    color = colors.fg.green + colors.bold
+    output = colors.reset + str(color) + '... ' + colors.reset + str(message) + colors.reset
+
+    if not get_settings() & USE_ANSI:
+        output = colors.filter(output)
+
+    sys.stdout.write(output)
+
+    return input()
+
+def confirm(default = 'y', message = 'Are you sure %s? '):
+    '''
+        Displays an "Are you sure" message, returns True for Y and False for N
+        message: The confirmation message, use %s for (y/n)
+        default: which to prefer-- y or n
+    '''
+
+    color = colors.fg.green + colors.bold
+
+    default = default.lower()
+    confirm = colors.bold
+    if default.startswith('y'):
+        confirm += '(Y/n)'
+    else:
+        confirm += '(y/N)'
+    confirm += colors.reset + color
+
+    output = colors.reset + str(color) + '... ' + colors.reset + str(message) + colors.reset
+
+    if not get_settings() & USE_ANSI:
+        output = colors.filter(output)
+
+    sys.stdout.write(output.replace('%s', confirm))
+
+    inp = input().lower()
+
+    if 'y' in inp:
+        return True
+    if 'n' in inp:
+        return False
+    else:
+        return default == 'y'
+
+# debug: when there is info that could be useful for debugging purposes only
+def debug(data, error = None, timestamp = True, prompt = True, sensitive = False, level = LEVEL_DEBUG):
+    if get_level() <= level:
+        log('/', data, timestamp = timestamp, prompt = prompt, sensitive = sensitive)
+    if not error is None:
+        debug('Error: ' + str(error) + parse_error())
+
+# info: when there is something to notify the user of, such as the success of a process
+def info(data, timestamp = False, prompt = True, sensitive = False, level = LEVEL_INFO):
+    if get_level() <= level:
+        log('+', data, colors.fg.green, timestamp = timestamp, prompt = prompt, sensitive = sensitive)
+
+# warn: when there is a potential for something bad to happen
+def warn(data, error = None, timestamp = True, prompt = True, sensitive = False, level = LEVEL_WARN):
+    if not error is None:
+        debug('Error: ' + str(error) + parse_error())
+    if get_level() <= level:
+        log('!', data, colors.fg.orange, timestamp = timestamp, prompt = prompt, sensitive = sensitive)
+
+# error: when only one function, module, or process of the program encountered a problem and must stop
+def error(data, error = None, timestamp = True, prompt = True, sensitive = False, level = LEVEL_ERROR):
+    if get_level() <= level:
+        log('-', data, colors.fg.red, timestamp = timestamp, fd = sys.stderr, prompt = prompt, sensitive = sensitive)
+    if not error is None:
+        debug('Error: ' + str(error) + parse_error())
+
+# fatal: when the something so bad has happened that the program must stop
+def fatal(data, error = None, timestamp=True, prompt = True, sensitive = False, level = LEVEL_FATAL):
+    if not error is None:
+        debug('Error: ' + str(error) + parse_error(), sensitive = sensitive)
+    if get_level() <= level:
+        log('#', data, colors.bg.red + colors.fg.green + colors.bold, timestamp = timestamp, fd = sys.stderr, prompt = prompt, sensitive = sensitive)
+
+# returns a formatted error message
+def parse_error():
+    details = traceback.extract_tb(sys.exc_info()[2])
+    output = ''
+
+    for line in details:
+        output += '\n    ... module %s in  %s:%i' % (line[2], line[0], line[1])
+
+    return output

onionr/netcontroller.py

@@ -0,0 +1,208 @@
+'''
+    Onionr - P2P Microblogging Platform & Social network
+
+    Netcontroller library, used to control/work with Tor/I2P and send requests through them
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+import subprocess, os, random, sys, logger, time, signal, config, base64, socket
+from stem.control import Controller
+from onionrblockapi import Block
+from dependencies import secrets
+from shutil import which
+
+def getOpenPort():
+    # taken from (but modified) https://stackoverflow.com/a/2838309
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    s.bind(("127.0.0.1",0))
+    s.listen(1)
+    port = s.getsockname()[1]
+    s.close()
+    return port
+
+def torBinary():
+    '''Return tor binary path or none if not exists'''
+    torPath = './tor'
+    if not os.path.exists(torPath):
+        torPath = which('tor')
+    return torPath
+
+class NetController:
+    '''
+        This class handles hidden service setup on Tor and I2P
+    '''
+
+    def __init__(self, hsPort, apiServerIP='127.0.0.1'):
+        # set data dir
+        self.dataDir = os.environ.get('ONIONR_HOME', os.environ.get('DATA_DIR', 'data/'))
+        if not self.dataDir.endswith('/'):
+            self.dataDir += '/'
+
+        self.torConfigLocation = self.dataDir + 'torrc'
+        self.readyState = False
+        self.socksPort = getOpenPort()
+        self.hsPort = hsPort
+        self._torInstnace = ''
+        self.myID = ''
+        self.apiServerIP = apiServerIP
+
+        if os.path.exists('./tor'):
+            self.torBinary = './tor'
+        elif os.path.exists('/usr/bin/tor'):
+            self.torBinary = '/usr/bin/tor'
+        else:
+            self.torBinary = 'tor'
+
+        config.reload()
+        '''
+            if os.path.exists(self.torConfigLocation):
+                torrc = open(self.torConfigLocation, 'r')
+                if not str(self.hsPort) in torrc.read():
+                    os.remove(self.torConfigLocation)
+                torrc.close()
+        '''
+
+        return
+
+    def generateTorrc(self):
+        '''
+            Generate a torrc file for our tor instance
+        '''
+        hsVer = '# v2 onions'
+        if config.get('tor.v3onions'):
+            hsVer = 'HiddenServiceVersion 3'
+            logger.debug('Using v3 onions')
+
+        if os.path.exists(self.torConfigLocation):
+            os.remove(self.torConfigLocation)
+
+        # Set the Tor control password. Meant to make it harder to manipulate our Tor instance
+        plaintext = base64.b64encode(os.urandom(50)).decode()
+        config.set('tor.controlpassword', plaintext, savefile=True)
+        config.set('tor.socksport', self.socksPort, savefile=True)
+
+        controlPort = getOpenPort()
+
+        config.set('tor.controlPort', controlPort, savefile=True)
+
+        hashedPassword = subprocess.Popen([self.torBinary, '--hash-password', plaintext], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        for line in iter(hashedPassword.stdout.readline, b''):
+            password = line.decode()
+            if 'warn' not in password:
+                break
+
+        torrcData = '''SocksPort ''' + str(self.socksPort) + '''
+DataDirectory ''' + self.dataDir + '''tordata/
+CookieAuthentication 1
+ControlPort ''' + str(controlPort) + '''
+HashedControlPassword ''' + str(password) + '''
+        '''
+        if config.get('general.security_level') == 0:
+            torrcData += '''\nHiddenServiceDir ''' + self.dataDir + '''hs/
+\n''' + hsVer + '''\n
+HiddenServicePort 80 ''' + self.apiServerIP + ''':''' + str(self.hsPort)
+
+        torrc = open(self.torConfigLocation, 'w')
+        torrc.write(torrcData)
+        torrc.close()
+
+        return
+
+    def startTor(self):
+        '''
+            Start Tor with onion service on port 80 & socks proxy on random port
+        '''
+
+        self.generateTorrc()
+
+        if os.path.exists('./tor'):
+            self.torBinary = './tor'
+        elif os.path.exists('/usr/bin/tor'):
+            self.torBinary = '/usr/bin/tor'
+        else:
+            self.torBinary = 'tor'
+
+        try:
+            tor = subprocess.Popen([self.torBinary, '-f', self.torConfigLocation], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        except FileNotFoundError:
+            logger.fatal("Tor was not found in your path or the Onionr directory. Please install Tor and try again.")
+            sys.exit(1)
+        else:
+            # Test Tor Version
+            torVersion = subprocess.Popen([self.torBinary, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            for line in iter(torVersion.stdout.readline, b''):
+                if 'Tor 0.2.' in line.decode():
+                    logger.error('Tor 0.3+ required')
+                    sys.exit(1)
+                    break
+            torVersion.kill()
+
+        # wait for tor to get to 100% bootstrap
+        try:
+            for line in iter(tor.stdout.readline, b''):
+                if 'Bootstrapped 100%: Done' in line.decode():
+                    break
+                elif 'Opening Socks listener' in line.decode():
+                    logger.debug(line.decode().replace('\n', ''))
+            else:
+                logger.fatal('Failed to start Tor. Maybe a stray instance of Tor used by Onionr is still running?')
+                return False
+        except KeyboardInterrupt:
+            logger.fatal('Got keyboard interrupt.', timestamp = False, level = logger.LEVEL_IMPORTANT)
+            return False
+
+        logger.debug('Finished starting Tor.', timestamp=True)
+        self.readyState = True
+
+        try:
+            myID = open(self.dataDir + 'hs/hostname', 'r')
+            self.myID = myID.read().replace('\n', '')
+            myID.close()
+        except FileNotFoundError:
+            self.myID = ""
+
+        torPidFile = open(self.dataDir + 'torPid.txt', 'w')
+        torPidFile.write(str(tor.pid))
+        torPidFile.close()
+
+        return True
+
+    def killTor(self):
+        '''
+            Properly kill tor based on pid saved to file
+        '''
+
+        try:
+            pid = open(self.dataDir + 'torPid.txt', 'r')
+            pidN = pid.read()
+            pid.close()
+        except FileNotFoundError:
+            return
+
+        try:
+            int(pidN)
+        except:
+            return
+
+        try:
+            os.kill(int(pidN), signal.SIGTERM)
+            os.remove(self.dataDir + 'torPid.txt')
+        except ProcessLookupError:
+            pass
+        except FileNotFoundError:
+            pass
+
+        return

onionr/onionr.py

@@ -0,0 +1,457 @@
+#!/usr/bin/env python3
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    Onionr is the name for both the protocol and the original/reference software.
+
+    Run with 'help' for usage.
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sys
+MIN_PY_VERSION = 6
+if sys.version_info[0] == 2 or sys.version_info[1] < MIN_PY_VERSION:
+    print('Error, Onionr requires Python 3.%s+' % (MIN_PY_VERSION,))
+    sys.exit(1)
+import os, base64, random, getpass, shutil, time, platform, datetime, re, json, getpass, sqlite3
+import webbrowser, uuid, signal
+from threading import Thread
+import api, core, config, logger, onionrplugins as plugins, onionrevents as events
+import onionrutils
+import netcontroller, onionrstorage
+from netcontroller import NetController
+from onionrblockapi import Block
+import onionrproofs, onionrexceptions, communicator, setupconfig
+from onionrusers import onionrusers
+import onionrcommands as commands # Many command definitions are here
+
+try:
+    from urllib3.contrib.socks import SOCKSProxyManager
+except ImportError:
+    raise Exception("You need the PySocks module (for use with socks5 proxy to use Tor)")
+
+ONIONR_TAGLINE = 'Anonymous P2P Platform - GPLv3 - https://Onionr.net'
+ONIONR_VERSION = '0.0.0' # for debugging and stuff
+ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION)
+API_VERSION = '0' # increments of 1; only change when something fundamental about how the API works changes. This way other nodes know how to communicate without learning too much information about you.
+
+class Onionr:
+    def __init__(self):
+        '''
+            Main Onionr class. This is for the CLI program, and does not handle much of the logic.
+            In general, external programs and plugins should not use this class.
+        '''
+        self.userRunDir = os.getcwd() # Directory user runs the program from
+        self.killed = False
+
+        if sys.argv[0] == os.path.basename(__file__):
+            try:
+                os.chdir(sys.path[0])
+            except FileNotFoundError:
+                pass
+
+        # set data dir
+        self.dataDir = os.environ.get('ONIONR_HOME', os.environ.get('DATA_DIR', 'data/'))
+        if not self.dataDir.endswith('/'):
+            self.dataDir += '/'
+
+        # set log file
+        logger.set_file(os.environ.get('LOG_DIR', 'data') + '/onionr.log')
+
+        # Load global configuration data
+        data_exists = Onionr.setupConfig(self.dataDir, self = self)
+
+        if netcontroller.torBinary() is None:
+            logger.error('Tor is not installed')
+            sys.exit(1)
+
+        # If data folder does not exist
+        if not data_exists:
+            if not os.path.exists(self.dataDir + 'blocks/'):
+                os.mkdir(self.dataDir + 'blocks/')
+
+        # Copy default plugins into plugins folder
+        if not os.path.exists(plugins.get_plugins_folder()):
+            if os.path.exists('static-data/default-plugins/'):
+                names = [f for f in os.listdir("static-data/default-plugins/")]
+                shutil.copytree('static-data/default-plugins/', plugins.get_plugins_folder())
+
+                # Enable plugins
+                for name in names:
+                    if not name in plugins.get_enabled_plugins():
+                        plugins.enable(name, self)
+
+        for name in plugins.get_enabled_plugins():
+            if not os.path.exists(plugins.get_plugin_data_folder(name)):
+                try:
+                    os.mkdir(plugins.get_plugin_data_folder(name))
+                except:
+                    plugins.disable(name, onionr = self, stop_event = False)
+
+        self.communicatorInst = None
+        self.onionrCore = core.Core()
+        self.onionrCore.onionrInst = self
+        #self.deleteRunFiles()
+        self.onionrUtils = onionrutils.OnionrUtils(self.onionrCore)
+
+        self.clientAPIInst = '' # Client http api instance
+        self.publicAPIInst = '' # Public http api instance
+
+        signal.signal(signal.SIGTERM, self.exitSigterm)
+
+        # Handle commands
+
+        self.debug = False # Whole application debugging
+
+        # Get configuration
+        if type(config.get('client.webpassword')) is type(None):
+            config.set('client.webpassword', base64.b16encode(os.urandom(32)).decode('utf-8'), savefile=True)
+        if type(config.get('client.client.port')) is type(None):
+            randomPort = netcontroller.getOpenPort()
+            config.set('client.client.port', randomPort, savefile=True)
+        if type(config.get('client.public.port')) is type(None):
+            randomPort = netcontroller.getOpenPort()
+            config.set('client.public.port', randomPort, savefile=True)
+        if type(config.get('client.participate')) is type(None):
+            config.set('client.participate', True, savefile=True)
+        if type(config.get('client.api_version')) is type(None):
+            config.set('client.api_version', API_VERSION, savefile=True)
+
+        self.cmds = commands.get_commands(self)
+        self.cmdhelp = commands.cmd_help
+
+        # initialize plugins
+        events.event('init', onionr = self, threaded = False)
+
+        command = ''
+        try:
+            command = sys.argv[1].lower()
+        except IndexError:
+            command = ''
+        finally:
+            self.execute(command)
+
+        return
+
+    def exitSigterm(self, signum, frame):
+        self.killed = True
+
+    def setupConfig(dataDir, self = None):
+        setupconfig.setup_config(dataDir, self)
+
+    def cmdHeader(self):
+        if len(sys.argv) >= 3:
+            self.header(logger.colors.fg.pink + sys.argv[2].replace('Onionr', logger.colors.bold + 'Onionr' + logger.colors.reset + logger.colors.fg.pink))
+        else:
+            self.header(None)
+
+    def header(self, message = logger.colors.fg.pink + logger.colors.bold + 'Onionr' + logger.colors.reset + logger.colors.fg.pink + ' has started.'):
+        if os.path.exists('static-data/header.txt') and logger.get_level() <= logger.LEVEL_INFO:
+            with open('static-data/header.txt', 'rb') as file:
+                # only to stdout, not file or log or anything
+                sys.stderr.write(file.read().decode().replace('P', logger.colors.fg.pink).replace('W', logger.colors.reset + logger.colors.bold).replace('G', logger.colors.fg.green).replace('\n', logger.colors.reset + '\n').replace('B', logger.colors.bold).replace('A', '%s' % API_VERSION).replace('V', ONIONR_VERSION))
+
+                if not message is None:
+                    logger.info(logger.colors.fg.lightgreen + '-> ' + str(message) + logger.colors.reset + logger.colors.fg.lightgreen + ' <-\n', sensitive=True)
+
+    def doExport(self, bHash):
+        exportDir = self.dataDir + 'block-export/'
+        if not os.path.exists(exportDir):
+            if os.path.exists(self.dataDir):
+                os.mkdir(exportDir)
+            else:
+                logger.error('Onionr Not initialized')
+        data = onionrstorage.getData(self.onionrCore, bHash)
+        with open('%s/%s.dat' % (exportDir, bHash), 'wb') as exportFile:
+            exportFile.write(data)
+
+    def deleteRunFiles(self):
+        try:
+            os.remove(self.onionrCore.publicApiHostFile)
+        except FileNotFoundError:
+            pass
+        try:
+            os.remove(self.onionrCore.privateApiHostFile)
+        except FileNotFoundError:
+            pass
+
+    def get_hostname(self):
+        try:
+            with open('./' + self.dataDir + 'hs/hostname', 'r') as hostname:
+                return hostname.read().strip()
+        except FileNotFoundError:
+            return "Not Generated"
+        except Exception:
+            return None
+
+    def getConsoleWidth(self):
+        '''
+            Returns an integer, the width of the terminal/cmd window
+        '''
+
+        columns = 80
+
+        try:
+            columns = int(os.popen('stty size', 'r').read().split()[1])
+        except:
+            # if it errors, it's probably windows, so default to 80.
+            pass
+
+        return columns
+
+    '''
+        THIS SECTION HANDLES THE COMMANDS
+    '''
+
+    def exportBlock(self):
+        exportDir = self.dataDir + 'block-export/'
+        try:
+            assert self.onionrUtils.validateHash(sys.argv[2])
+        except (IndexError, AssertionError):
+            logger.error('No valid block hash specified.')
+            sys.exit(1)
+        else:
+            bHash = sys.argv[2]
+            self.doExport(bHash)
+
+    def showDetails(self):
+        commands.onionrstatistics.show_details(self)
+
+    def openHome(self):
+        commands.open_home(self)
+
+    def addID(self):
+        commands.pubkeymanager.add_ID(self)
+
+    def changeID(self):
+        commands.pubkeymanager.change_ID(self)
+
+    def getCommands(self):
+        return self.cmds
+
+    def friendCmd(self):
+        '''List, add, or remove friend(s)
+        Changes their peer DB entry.
+        '''
+        commands.pubkeymanager.friend_command(self)
+
+    def banBlock(self):
+        try:
+            ban = sys.argv[2]
+        except IndexError:
+            ban = logger.readline('Enter a block hash:')
+        if self.onionrUtils.validateHash(ban):
+            if not self.onionrCore._blacklist.inBlacklist(ban):
+                try:
+                    self.onionrCore._blacklist.addToDB(ban)
+                    self.onionrCore.removeBlock(ban)
+                except Exception as error:
+                    logger.error('Could not blacklist block', error=error)
+                else:
+                    logger.info('Block blacklisted')
+            else:
+                logger.warn('That block is already blacklisted')
+        else:
+            logger.error('Invalid block hash')
+
+    def listConn(self):
+        commands.onionrstatistics.show_peers(self)
+
+    def listPeers(self):
+        logger.info('Peer transport address list:')
+        for i in self.onionrCore.listAdders():
+            logger.info(i)
+
+    def getWebPassword(self):
+        return config.get('client.webpassword')
+
+    def printWebPassword(self):
+        logger.info(self.getWebPassword(), sensitive = True)
+
+    def getHelp(self):
+        return self.cmdhelp
+
+    def addCommand(self, command, function):
+        self.cmds[str(command).lower()] = function
+
+    def addHelp(self, command, description):
+        self.cmdhelp[str(command).lower()] = str(description)
+
+    def delCommand(self, command):
+        return self.cmds.pop(str(command).lower(), None)
+
+    def delHelp(self, command):
+        return self.cmdhelp.pop(str(command).lower(), None)
+
+    def configure(self):
+        '''
+            Displays something from the configuration file, or sets it
+        '''
+
+        if len(sys.argv) >= 4:
+            config.reload()
+            config.set(sys.argv[2], sys.argv[3], True)
+            logger.debug('Configuration file updated.')
+        elif len(sys.argv) >= 3:
+            config.reload()
+            logger.info(logger.colors.bold + sys.argv[2] + ': ' + logger.colors.reset + str(config.get(sys.argv[2], logger.colors.fg.red + 'Not set.')))
+        else:
+            logger.info(logger.colors.bold + 'Get a value: ' + logger.colors.reset + sys.argv[0] + ' ' + sys.argv[1] + ' <key>')
+            logger.info(logger.colors.bold + 'Set a value: ' + logger.colors.reset + sys.argv[0] + ' ' + sys.argv[1] + ' <key> <value>')
+
+    def execute(self, argument):
+        '''
+            Executes a command
+        '''
+
+        argument = argument[argument.startswith('--') and len('--'):] # remove -- if it starts with it
+
+        # define commands
+        commands = self.getCommands()
+
+        command = commands.get(argument, self.notFound)
+        command()
+
+    def version(self, verbosity = 5, function = logger.info):
+        '''
+            Displays the Onionr version
+        '''
+
+        function('Onionr v%s (%s) (API v%s)' % (ONIONR_VERSION, platform.machine(), API_VERSION))
+        if verbosity >= 1:
+            function(ONIONR_TAGLINE)
+        if verbosity >= 2:
+            function('Running on %s %s' % (platform.platform(), platform.release()))
+            function('Onionr data dir: %s' % self.dataDir)
+
+    def doPEX(self):
+        '''make communicator do pex'''
+        logger.info('Sending pex to command queue...')
+        self.onionrCore.daemonQueueAdd('pex')
+
+    def listKeys(self):
+        '''
+            Displays a list of keys (used to be called peers) (?)
+        '''
+        logger.info('%sPublic keys in database: \n%s%s' % (logger.colors.fg.lightgreen, logger.colors.fg.green, '\n'.join(self.onionrCore.listPeers())))
+
+    def addPeer(self):
+        '''
+            Adds a peer (?)
+        '''
+        commands.keyadders.add_peer(self)
+
+    def addAddress(self):
+        '''
+            Adds a Onionr node address
+        '''
+        commands.keyadders.add_address(self)
+
+    def enablePlugin(self):
+        '''
+            Enables and starts the given plugin
+        '''
+        commands.plugincommands.enable_plugin(self)
+
+    def disablePlugin(self):
+        '''
+            Disables and stops the given plugin
+        '''
+        commands.plugincommands.disable_plugin(self)
+
+    def reloadPlugin(self):
+        '''
+            Reloads (stops and starts) all plugins, or the given plugin
+        '''
+        commands.plugincommands.reload_plugin(self)
+
+    def createPlugin(self):
+        '''
+            Creates the directory structure for a plugin name
+        '''
+        commands.plugincommands.create_plugin(self)
+
+    def notFound(self):
+        '''
+            Displays a "command not found" message
+        '''
+
+        logger.error('Command not found.', timestamp = False)
+
+    def showHelpSuggestion(self):
+        '''
+            Displays a message suggesting help
+        '''
+        if __name__ == '__main__':
+            logger.info('Do ' + logger.colors.bold + sys.argv[0] + ' --help' + logger.colors.reset + logger.colors.fg.green + ' for Onionr help.')
+
+    def start(self, input = False, override = False):
+        '''
+            Starts the Onionr daemon
+        '''
+        commands.daemonlaunch.start(self, input, override)
+
+    def setClientAPIInst(self, inst):
+        self.clientAPIInst = inst
+
+    def getClientApi(self):
+        while self.clientAPIInst == '':
+            time.sleep(0.5)
+        return self.clientAPIInst
+
+    def daemon(self):
+        '''
+            Starts the Onionr communication daemon
+        '''
+        commands.daemonlaunch.daemon(self)
+
+    def killDaemon(self):
+        '''
+            Shutdown the Onionr daemon
+        '''
+        commands.daemonlaunch.kill_daemon(self)
+
+    def showStats(self):
+        '''
+            Displays statistics and exits
+        '''
+        commands.onionrstatistics.show_stats(self)
+
+    def showHelp(self, command = None):
+        '''
+            Show help for Onionr
+        '''
+        commands.show_help(self, command)
+
+    def getFile(self):
+        '''
+            Get a file from onionr blocks
+        '''
+        commands.filecommands.getFile(self)
+
+    def addWebpage(self):
+        '''
+            Add a webpage to the onionr network
+        '''
+        self.addFile(singleBlock=True, blockType='html')
+
+    def addFile(self, singleBlock=False, blockType='bin'):
+        '''
+            Adds a file to the onionr network
+        '''
+        commands.filecommands.add_file(self, singleBlock, blockType)
+
+if __name__ == "__main__":
+    Onionr()

onionr/onionrblacklist.py

@@ -0,0 +1,122 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This file handles maintenence of a blacklist database, for blocks and peers
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sqlite3, os, logger
+class OnionrBlackList:
+    def __init__(self, coreInst):
+        self.blacklistDB = coreInst.dataDir + 'blacklist.db'
+        self._core = coreInst
+
+        if not os.path.exists(self.blacklistDB):
+            self.generateDB()
+        return
+
+    def inBlacklist(self, data):
+        hashed = self._core._utils.bytesToStr(self._core._crypto.sha3Hash(data))
+        retData = False
+
+        if not hashed.isalnum():
+            raise Exception("Hashed data is not alpha numeric")
+        if len(hashed) > 64:
+            raise Exception("Hashed data is too large")
+
+        for i in self._dbExecute("SELECT * FROM blacklist WHERE hash = ?", (hashed,)):
+            retData = True # this only executes if an entry is present by that hash
+            break
+
+        return retData
+
+    def _dbExecute(self, toExec, params = ()):
+        conn = sqlite3.connect(self.blacklistDB)
+        c = conn.cursor()
+        retData = c.execute(toExec, params)
+        conn.commit()
+        return retData
+
+    def deleteBeforeDate(self, date):
+        # TODO, delete blacklist entries before date
+        return
+
+    def deleteExpired(self, dataType=0):
+        '''Delete expired entries'''
+        deleteList = []
+        curTime = self._core._utils.getEpoch()
+
+        try:
+            int(dataType)
+        except AttributeError:
+            raise TypeError("dataType must be int")
+
+        for i in self._dbExecute('SELECT * FROM blacklist WHERE dataType = ?', (dataType,)):
+            if i[1] == dataType:
+                if (curTime - i[2]) >= i[3]:
+                    deleteList.append(i[0])
+
+        for thing in deleteList:
+            self._dbExecute("DELETE FROM blacklist WHERE hash = ?", (thing,))
+
+    def generateDB(self):
+        self._dbExecute('''CREATE TABLE blacklist(
+            hash text primary key not null,
+            dataType int,
+            blacklistDate int,
+            expire int
+            );
+        ''')
+        return
+
+    def clearDB(self):
+        self._dbExecute('''DELETE FROM blacklist;''')
+
+    def getList(self):
+        data = self._dbExecute('SELECT * FROM blacklist')
+        myList = []
+        for i in data:
+            myList.append(i[0])
+        return myList
+
+    def addToDB(self, data, dataType=0, expire=0):
+        '''Add to the blacklist. Intended to be block hash, block data, peers, or transport addresses
+        0=block
+        1=peer
+        2=pubkey
+        '''
+        # we hash the data so we can remove data entirely from our node's disk
+        hashed = self._core._utils.bytesToStr(self._core._crypto.sha3Hash(data))
+        if len(hashed) > 64:
+            raise Exception("Hashed data is too large")
+
+        if not hashed.isalnum():
+            raise Exception("Hashed data is not alpha numeric")
+        try:
+            int(dataType)
+        except ValueError:
+            raise Exception("dataType is not int")
+        try:
+            int(expire)
+        except ValueError:
+            raise Exception("expire is not int")
+        if self.inBlacklist(hashed):
+            return
+        insert = (hashed,)
+        blacklistDate = self._core._utils.getEpoch()
+        try:
+            self._dbExecute("INSERT INTO blacklist (hash, dataType, blacklistDate, expire) VALUES(?, ?, ?, ?);", (str(hashed), dataType, blacklistDate, expire))
+        except sqlite3.IntegrityError:
+            pass

onionr/onionrblockapi.py

@@ -0,0 +1,720 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This file contains the OnionrBlocks class which is a class for working with Onionr blocks
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+import core as onionrcore, logger, config, onionrexceptions, nacl.exceptions
+import json, os, sys, datetime, base64, onionrstorage
+from onionrusers import onionrusers
+
+class Block:
+    blockCacheOrder = list() # NEVER write your own code that writes to this!
+    blockCache = dict() # should never be accessed directly, look at Block.getCache()
+
+    def __init__(self, hash = None, core = None, type = None, content = None, expire=None, decrypt=False, bypassReplayCheck=False):
+        # take from arguments
+        # sometimes people input a bytes object instead of str in `hash`
+        if (not hash is None) and isinstance(hash, bytes):
+            hash = hash.decode()
+
+        self.hash = hash
+        self.core = core
+        self.btype = type
+        self.bcontent = content
+        self.expire = expire
+        self.bypassReplayCheck = bypassReplayCheck
+
+        # initialize variables
+        self.valid = True
+        self.raw = None
+        self.signed = False
+        self.signature = None
+        self.signedData = None
+        self.blockFile = None
+        self.parent = None
+        self.bheader = {}
+        self.bmetadata = {}
+        self.isEncrypted = False
+        self.decrypted = False
+        self.signer = None
+        self.validSig = False
+        self.autoDecrypt = decrypt
+
+        # handle arguments
+        if self.getCore() is None:
+            self.core = onionrcore.Core()
+
+        self.update()
+
+    def decrypt(self, encodedData = True):
+        '''
+            Decrypt a block, loading decrypted data into their vars
+        '''
+
+        if self.decrypted:
+            return True
+        retData = False
+        core = self.getCore()
+        # decrypt data
+        if self.getHeader('encryptType') == 'asym':
+            try:
+                self.bcontent = core._crypto.pubKeyDecrypt(self.bcontent, encodedData=encodedData)
+                bmeta = core._crypto.pubKeyDecrypt(self.bmetadata, encodedData=encodedData)
+                try:
+                    bmeta = bmeta.decode()
+                except AttributeError:
+                    # yet another bytes fix
+                    pass
+                self.bmetadata = json.loads(bmeta)
+                self.signature = core._crypto.pubKeyDecrypt(self.signature, encodedData=encodedData)
+                self.signer = core._crypto.pubKeyDecrypt(self.signer, encodedData=encodedData)
+                self.bheader['signer'] = self.signer.decode()
+                self.signedData =  json.dumps(self.bmetadata) + self.bcontent.decode()
+
+                # Check for replay attacks
+                try:
+                    if self.core._utils.getEpoch() - self.core.getBlockDate(self.hash) < 60:
+                        assert self.core._crypto.replayTimestampValidation(self.bmetadata['rply'])
+                except (AssertionError, KeyError) as e:
+                    if not self.bypassReplayCheck:
+                        # Zero out variables to prevent reading of replays
+                        self.bmetadata = {}
+                        self.signer = ''
+                        self.bheader['signer'] = ''
+                        self.signedData = ''
+                        self.signature = ''
+                        raise onionrexceptions.ReplayAttack('Signature is too old. possible replay attack')
+                try:
+                    assert self.bmetadata['forwardEnc'] is True
+                except (AssertionError, KeyError) as e:
+                    pass
+                else:
+                    try:
+                        self.bcontent = onionrusers.OnionrUser(self.getCore(), self.signer).forwardDecrypt(self.bcontent)
+                    except (onionrexceptions.DecryptionError, nacl.exceptions.CryptoError) as e:
+                        logger.error(str(e))
+                        pass
+            except nacl.exceptions.CryptoError:
+                pass
+                #logger.debug('Could not decrypt block. Either invalid key or corrupted data')
+            except onionrexceptions.ReplayAttack:
+                logger.warn('%s is possibly a replay attack' % (self.hash,))
+            else:
+                retData = True
+                self.decrypted = True
+        else:
+            logger.warn('symmetric decryption is not yet supported by this API')
+        return retData
+
+    def verifySig(self):
+        '''
+            Verify if a block's signature is signed by its claimed signer
+        '''
+
+        core = self.getCore()
+
+        if core._crypto.edVerify(data=self.signedData, key=self.signer, sig=self.signature, encodedData=True):
+            self.validSig = True
+        else:
+            self.validSig = False
+        return self.validSig
+
+
+    def update(self, data = None, file = None):
+        '''
+            Loads data from a block in to the current object.
+
+            Inputs:
+            - data (str):
+              - if None: will load from file by hash
+              - else: will load from `data` string
+            - file (str):
+              - if None: will load from file specified in this parameter
+              - else: will load from wherever block is stored by hash
+
+            Outputs:
+            - (bool): indicates whether or not the operation was successful
+        '''
+        try:
+            # import from string
+            blockdata = data
+
+            # import from file
+            if blockdata is None:
+                try:
+                    blockdata = onionrstorage.getData(self.core, self.getHash()).decode()
+                except AttributeError:
+                    raise onionrexceptions.NoDataAvailable('Block does not exist')
+            else:
+                self.blockFile = None
+            # parse block
+            self.raw = str(blockdata)
+            self.bheader = json.loads(self.getRaw()[:self.getRaw().index('\n')])
+            self.bcontent = self.getRaw()[self.getRaw().index('\n') + 1:]
+            if ('encryptType' in self.bheader) and (self.bheader['encryptType'] in ('asym', 'sym')):
+                self.bmetadata = self.getHeader('meta', None)
+                self.isEncrypted = True
+            else:
+                self.bmetadata = json.loads(self.getHeader('meta', None))
+            self.parent = self.getMetadata('parent', None)
+            self.btype = self.getMetadata('type', None)
+            self.signed = ('sig' in self.getHeader() and self.getHeader('sig') != '')
+            # TODO: detect if signer is hash of pubkey or not
+            self.signer = self.getHeader('signer', None)
+            self.signature = self.getHeader('sig', None)
+            # signed data is jsonMeta + block content (no linebreak)
+            self.signedData = (None if not self.isSigned() else self.getHeader('meta') + self.getContent())
+            self.date = self.getCore().getBlockDate(self.getHash())
+            self.claimedTime = self.getHeader('time', None)
+
+            if not self.getDate() is None:
+                self.date = datetime.datetime.fromtimestamp(self.getDate())
+
+            self.valid = True
+
+            if len(self.getRaw()) <= config.get('allocations.blockCache', 500000):
+                self.cache()
+            
+            if self.autoDecrypt:
+                self.decrypt()
+
+            return True
+        except Exception as e:
+            logger.warn('Failed to parse block %s.' % self.getHash(), error = e, timestamp = False)
+
+            # if block can't be parsed, it's a waste of precious space. Throw it away.
+            if not self.delete():
+                logger.warn('Failed to delete invalid block %s.' % self.getHash(), error = e)
+            else:
+                logger.debug('Deleted invalid block %s.' % self.getHash(), timestamp = False)
+
+        self.valid = False
+        return False
+
+    def delete(self):
+        '''
+            Deletes the block's file and records, if they exist
+
+            Outputs:
+            - (bool): whether or not the operation was successful
+        '''
+
+        if self.exists():
+            os.remove(self.getBlockFile())
+            self.getCore().removeBlock(self.getHash())
+            return True
+        return False
+
+    def save(self, sign = False, recreate = True):
+        '''
+            Saves a block to file and imports it into Onionr
+
+            Inputs:
+            - sign (bool): whether or not to sign the block before saving
+            - recreate (bool): if the block already exists, whether or not to recreate the block and save under a new hash
+
+            Outputs:
+            - (bool): whether or not the operation was successful
+        '''
+
+        try:
+            if self.isValid() is True:
+                '''
+                if (not self.getBlockFile() is None) and (recreate is True):
+                    onionrstorage.store(self.core, self.getRaw().encode())
+                    #with open(self.getBlockFile(), 'wb') as blockFile:
+                    #    blockFile.write(self.getRaw().encode())
+                else:
+                '''
+                self.hash = self.getCore().insertBlock(self.getRaw(), header = self.getType(), sign = sign, meta = self.getMetadata(), expire = self.getExpire())
+                if self.hash != False:
+                    self.update()
+
+                return self.getHash()
+            else:
+                logger.warn('Not writing block; it is invalid.')
+        except Exception as e:
+            logger.error('Failed to save block.', error = e, timestamp = False)
+
+        return False
+
+    # getters
+
+    def getExpire(self):
+        '''
+            Returns the expire time for a block
+
+            Outputs:
+            - (int): the expire time for a block, or None
+        '''
+        return self.expire
+
+    def getHash(self):
+        '''
+            Returns the hash of the block if saved to file
+
+            Outputs:
+            - (str): the hash of the block, or None
+        '''
+
+        return self.hash
+
+    def getCore(self):
+        '''
+            Returns the Core instance being used by the Block
+
+            Outputs:
+            - (Core): the Core instance
+        '''
+
+        return self.core
+
+    def getType(self):
+        '''
+            Returns the type of the block
+
+            Outputs:
+            - (str): the type of the block
+        '''
+        return self.btype
+
+    def getRaw(self):
+        '''
+            Returns the raw contents of the block, if saved to file
+
+            Outputs:
+            - (str): the raw contents of the block, or None
+        '''
+
+        return str(self.raw)
+
+    def getHeader(self, key = None, default = None):
+        '''
+            Returns the header information
+
+            Inputs:
+            - key (str): only returns the value of the key in the header
+
+            Outputs:
+            - (dict/str): either the whole header as a dict, or one value
+        '''
+
+        if not key is None:
+            if key in self.getHeader():
+                return self.getHeader()[key]
+            return default
+        return self.bheader
+
+    def getMetadata(self, key = None, default = None):
+        '''
+            Returns the metadata information
+
+            Inputs:
+            - key (str): only returns the value of the key in the metadata
+
+            Outputs:
+            - (dict/str): either the whole metadata as a dict, or one value
+        '''
+
+        if not key is None:
+            if key in self.getMetadata():
+                return self.getMetadata()[key]
+            return default
+        return self.bmetadata
+
+    def getContent(self):
+        '''
+            Returns the contents of the block
+
+            Outputs:
+            - (str): the contents of the block
+        '''
+
+        return str(self.bcontent)
+
+    def getParent(self):
+        '''
+            Returns the Block's parent Block, or None
+
+            Outputs:
+            - (Block): the Block's parent
+        '''
+
+        if type(self.parent) == str:
+            if self.parent == self.getHash():
+                self.parent = self
+            elif Block.exists(self.parent):
+                self.parent = Block(self.getMetadata('parent'), core = self.getCore())
+            else:
+                self.parent = None
+
+        return self.parent
+
+    def getDate(self):
+        '''
+            Returns the date that the block was received, if loaded from file
+
+            Outputs:
+            - (datetime): the date that the block was received
+        '''
+
+        return self.date
+
+    def getBlockFile(self):
+        '''
+            Returns the location of the block file if it is saved
+
+            Outputs:
+            - (str): the location of the block file, or None
+        '''
+
+        return self.blockFile
+
+    def isValid(self):
+        '''
+            Checks if the block is valid
+
+            Outputs:
+            - (bool): whether or not the block is valid
+        '''
+
+        return self.valid
+
+    def isSigned(self):
+        '''
+            Checks if the block was signed
+
+            Outputs:
+            - (bool): whether or not the block is signed
+        '''
+
+        return self.signed
+
+    def getSignature(self):
+        '''
+            Returns the base64-encoded signature
+
+            Outputs:
+            - (str): the signature, or None
+        '''
+
+        return self.signature
+
+    def getSignedData(self):
+        '''
+            Returns the data that was signed
+
+            Outputs:
+            - (str): the data that was signed, or None
+        '''
+
+        return self.signedData
+
+    def isSigner(self, signer, encodedData = True):
+        '''
+            Checks if the block was signed by the signer inputted
+
+            Inputs:
+            - signer (str): the public key of the signer to check against
+            - encodedData (bool): whether or not the `signer` argument is base64 encoded
+
+            Outputs:
+            - (bool): whether or not the signer of the block is the signer inputted
+        '''
+
+        try:
+            if (not self.isSigned()) or (not self.getCore()._utils.validatePubKey(signer)):
+                return False
+
+            return bool(self.getCore()._crypto.edVerify(self.getSignedData(), signer, self.getSignature(), encodedData = encodedData))
+        except:
+            return False
+
+    # setters
+
+    def setType(self, btype):
+        '''
+            Sets the type of the block
+
+            Inputs:
+            - btype (str): the type of block to be set to
+
+            Outputs:
+            - (Block): the Block instance
+        '''
+
+        self.btype = btype
+        return self
+
+    def setMetadata(self, key, val):
+        '''
+            Sets a custom metadata value
+
+            Metadata should not store block-specific data structures.
+
+            Inputs:
+            - key (str): the key
+            - val: the value (type is irrelevant)
+
+            Outputs:
+            - (Block): the Block instance
+        '''
+
+        if key == 'parent' and (not val is None) and (not val == self.getParent().getHash()):
+            self.setParent(val)
+        else:
+            self.bmetadata[key] = val
+        return self
+
+    def setContent(self, bcontent):
+        '''
+            Sets the contents of the block
+
+            Inputs:
+            - bcontent (str): the contents to be set to
+
+            Outputs:
+            - (Block): the Block instance
+        '''
+
+        self.bcontent = str(bcontent)
+        return self
+
+    def setParent(self, parent):
+        '''
+            Sets the Block's parent
+
+            Inputs:
+            - parent (Block/str): the Block's parent, to be stored in metadata
+
+            Outputs:
+            - (Block): the Block instance
+        '''
+
+        if type(parent) == str:
+            parent = Block(parent, core = self.getCore())
+
+        self.parent = parent
+        self.setMetadata('parent', (None if parent is None else self.getParent().getHash()))
+        return self
+
+    # static functions
+
+    def getBlocks(type = None, signer = None, signed = None, parent = None, reverse = False, limit = None, core = None):
+        '''
+            Returns a list of Block objects based on supplied filters
+
+            Inputs:
+            - type (str): filters by block type
+            - signer (str/list): filters by signer (one in the list has to be a signer)
+            - signed (bool): filters out by whether or not the block is signed
+            - reverse (bool): reverses the list if True
+            - core (Core): lets you optionally supply a core instance so one doesn't need to be started
+
+            Outputs:
+            - (list): a list of Block objects that match the input
+        '''
+
+        try:
+            core = (core if not core is None else onionrcore.Core())
+
+            if (not parent is None) and (not isinstance(parent, Block)):
+                parent = Block(hash = parent, core = core)
+
+            relevant_blocks = list()
+            blocks = (core.getBlockList() if type is None else core.getBlocksByType(type))
+
+            for block in blocks:
+                if Block.exists(block):
+                    block = Block(block, core = core)
+
+                    relevant = True
+
+                    if (not signed is None) and (block.isSigned() != bool(signed)):
+                        relevant = False
+                    if not signer is None:
+                        if isinstance(signer, (str,)):
+                            signer = [signer]
+                        if isinstance(signer, (bytes,)):
+                            signer = [signer.decode()]
+
+                        isSigner = False
+                        for key in signer:
+                            if block.isSigner(key):
+                                isSigner = True
+                                break
+
+                        if not isSigner:
+                            relevant = False
+
+                    if not parent is None:
+                        blockParent = block.getParent()
+
+                        if blockParent is None:
+                            relevant = False
+                        else:
+                            relevant = parent.getHash() == blockParent.getHash()
+
+                    if relevant and (limit is None or len(relevant_Blocks) <= int(limit)):
+                        relevant_blocks.append(block)
+
+            if bool(reverse):
+                relevant_blocks.reverse()
+
+            return relevant_blocks
+        except Exception as e:
+            logger.debug('Failed to get blocks.', error = e)
+
+        return list()
+
+    def mergeChain(child, file = None, maximumFollows = 1000, core = None):
+        '''
+            Follows a child Block to its root parent Block, merging content
+
+            Inputs:
+            - child (str/Block): the child Block to be followed
+            - file (str/file): the file to write the content to, instead of returning it
+            - maximumFollows (int): the maximum number of Blocks to follow
+        '''
+
+        # validate data and instantiate Core
+        core = (core if not core is None else onionrcore.Core())
+        maximumFollows = max(0, maximumFollows)
+
+        # type conversions
+        if type(child) == list:
+            child = child[-1]
+        if type(child) == str:
+            child = Block(child)
+        if (not file is None) and (type(file) == str):
+            file = open(file, 'ab')
+
+        # only store hashes to avoid intensive memory usage
+        blocks = [child.getHash()]
+
+        # generate a list of parent Blocks
+        while True:
+            # end if the maximum number of follows has been exceeded
+            if len(blocks) - 1 >= maximumFollows:
+                break
+
+            block = Block(blocks[-1], core = core).getParent()
+
+            # end if there is no parent Block
+            if block is None:
+                break
+
+            # end if the Block is pointing to a previously parsed Block
+            if block.getHash() in blocks:
+                break
+
+            # end if the block is not valid
+            if not block.isValid():
+                break
+
+            blocks.append(block.getHash())
+
+        buffer = b''
+
+        # combine block contents
+        for hash in blocks:
+            block = Block(hash, core = core)
+            contents = block.getContent()
+            contents = base64.b64decode(contents.encode())
+
+            if file is None:
+                try:
+                    buffer += contents.encode()
+                except AttributeError:
+                    buffer += contents
+            else:
+                file.write(contents)
+        if file is not None:
+            file.close()
+
+        return (None if not file is None else buffer)
+
+    def exists(bHash):
+        '''
+            Checks if a block is saved to file or not
+
+            Inputs:
+            - hash (str/Block):
+              - if (Block): check if this block is saved to file
+              - if (str): check if a block by this hash is in file
+
+            Outputs:
+            - (bool): whether or not the block file exists
+        '''
+
+        # no input data? scrap it.
+        if bHash is None:
+            return False
+        '''
+        if type(hash) == Block:
+            blockfile = hash.getBlockFile()
+        else:
+            blockfile = onionrcore.Core().dataDir + 'blocks/%s.dat' % hash
+        '''
+        if isinstance(bHash, Block):
+            bHash = bHash.getHash()
+        
+        ret = isinstance(onionrstorage.getData(onionrcore.Core(), bHash), type(None))
+
+        return not ret
+
+    def getCache(hash = None):
+        # give a list of the hashes of the cached blocks
+        if hash is None:
+            return list(Block.blockCache.keys())
+
+        # if they inputted self or a Block, convert to hash
+        if type(hash) == Block:
+            hash = hash.getHash()
+
+        # just to make sure someone didn't put in a bool or something lol
+        hash = str(hash)
+
+        # if it exists, return its content
+        if hash in Block.getCache():
+            return Block.blockCache[hash]
+
+        return None
+
+    def cache(block, override = False):
+        # why even bother if they're giving bad data?
+        if not type(block) == Block:
+            return False
+
+        # only cache if written to file
+        if block.getHash() is None:
+            return False
+
+        # if it's already cached, what are we here for?
+        if block.getHash() in Block.getCache() and not override:
+            return False
+
+        # dump old cached blocks if the size exceeds the maximum
+        if sys.getsizeof(Block.blockCacheOrder) >= config.get('allocations.block_cache_total', 50000000): # 50MB default cache size
+            del Block.blockCache[blockCacheOrder.pop(0)]
+
+        # cache block content
+        Block.blockCache[block.getHash()] = block.getRaw()
+        Block.blockCacheOrder.append(block.getHash())
+
+        return True

onionr/onionrcommands/__init__.py

@@ -0,0 +1,172 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This module defines commands for CLI usage
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+import webbrowser, sys
+import logger
+from . import pubkeymanager, onionrstatistics, daemonlaunch, filecommands, plugincommands, keyadders
+
+def show_help(o_inst, command):
+
+    helpmenu = o_inst.getHelp()
+
+    if command is None and len(sys.argv) >= 3:
+        for cmd in sys.argv[2:]:
+            o_inst.showHelp(cmd)
+    elif not command is None:
+        if command.lower() in helpmenu:
+            logger.info(logger.colors.bold + command  + logger.colors.reset + logger.colors.fg.blue + ' : ' + logger.colors.reset +  helpmenu[command.lower()], timestamp = False)
+        else:
+            logger.warn(logger.colors.bold + command  + logger.colors.reset + logger.colors.fg.blue + ' : ' + logger.colors.reset + 'No help menu entry was found', timestamp = False)
+    else:
+        o_inst.version(0)
+        for command, helpmessage in helpmenu.items():
+            o_inst.showHelp(command)
+
+def open_home(o_inst):
+    try:
+        url = o_inst.onionrUtils.getClientAPIServer()
+    except FileNotFoundError:
+        logger.error('Onionr seems to not be running (could not get api host)')
+    else:
+        url = 'http://%s/#%s' % (url, o_inst.onionrCore.config.get('client.webpassword'))
+        logger.info('If Onionr does not open automatically, use this URL:\n\n%s' % url)
+        webbrowser.open_new_tab(url)
+
+def get_commands(onionr_inst):
+    return {'': onionr_inst.showHelpSuggestion,
+    'help': onionr_inst.showHelp,
+    'version': onionr_inst.version,
+    'header': onionr_inst.cmdHeader,
+    'config': onionr_inst.configure,
+    'start': onionr_inst.start,
+    'stop': onionr_inst.killDaemon,
+    'status': onionr_inst.showStats,
+    'statistics': onionr_inst.showStats,
+    'stats': onionr_inst.showStats,
+    'details' : onionr_inst.showDetails,
+    'detail' : onionr_inst.showDetails,
+    'show-details' : onionr_inst.showDetails,
+    'show-detail' : onionr_inst.showDetails,
+    'showdetails' : onionr_inst.showDetails,
+    'showdetail' : onionr_inst.showDetails,
+    'get-details' : onionr_inst.showDetails,
+    'get-detail' : onionr_inst.showDetails,
+    'getdetails' : onionr_inst.showDetails,
+    'getdetail' : onionr_inst.showDetails,
+
+    'enable-plugin': onionr_inst.enablePlugin,
+    'enplugin': onionr_inst.enablePlugin,
+    'enableplugin': onionr_inst.enablePlugin,
+    'enmod': onionr_inst.enablePlugin,
+    'disable-plugin': onionr_inst.disablePlugin,
+    'displugin': onionr_inst.disablePlugin,
+    'disableplugin': onionr_inst.disablePlugin,
+    'dismod': onionr_inst.disablePlugin,
+    'reload-plugin': onionr_inst.reloadPlugin,
+    'reloadplugin': onionr_inst.reloadPlugin,
+    'reload-plugins': onionr_inst.reloadPlugin,
+    'reloadplugins': onionr_inst.reloadPlugin,
+    'create-plugin': onionr_inst.createPlugin,
+    'createplugin': onionr_inst.createPlugin,
+    'plugin-create': onionr_inst.createPlugin,
+
+    'listkeys': onionr_inst.listKeys,
+    'list-keys': onionr_inst.listKeys,
+
+    'addpeer': onionr_inst.addPeer,
+    'add-peer': onionr_inst.addPeer,
+    'add-address': onionr_inst.addAddress,
+    'add-addr': onionr_inst.addAddress,
+    'addaddr': onionr_inst.addAddress,
+    'addaddress': onionr_inst.addAddress,
+    'list-peers': onionr_inst.listPeers,
+
+    'blacklist-block': onionr_inst.banBlock,
+
+    'add-file': onionr_inst.addFile,
+    'addfile': onionr_inst.addFile,
+    'addhtml': onionr_inst.addWebpage,
+    'add-html': onionr_inst.addWebpage,
+    'add-site': onionr_inst.addWebpage,
+    'addsite': onionr_inst.addWebpage,
+
+    'openhome': onionr_inst.openHome,
+    'open-home': onionr_inst.openHome,
+
+    'export-block': onionr_inst.exportBlock,
+    'exportblock': onionr_inst.exportBlock,
+
+    'get-file': onionr_inst.getFile,
+    'getfile': onionr_inst.getFile,
+
+    'listconn': onionr_inst.listConn,
+    'list-conn': onionr_inst.listConn,
+
+    'import-blocks': onionr_inst.onionrUtils.importNewBlocks,
+    'importblocks': onionr_inst.onionrUtils.importNewBlocks,
+
+    'introduce': onionr_inst.onionrCore.introduceNode,
+    'pex': onionr_inst.doPEX,
+
+    'getpassword': onionr_inst.printWebPassword,
+    'get-password': onionr_inst.printWebPassword,
+    'getpwd': onionr_inst.printWebPassword,
+    'get-pwd': onionr_inst.printWebPassword,
+    'getpass': onionr_inst.printWebPassword,
+    'get-pass': onionr_inst.printWebPassword,
+    'getpasswd': onionr_inst.printWebPassword,
+    'get-passwd': onionr_inst.printWebPassword,
+
+    'friend': onionr_inst.friendCmd,
+    'addid': onionr_inst.addID,
+    'add-id': onionr_inst.addID,
+    'change-id': onionr_inst.changeID
+    }
+
+cmd_help = {
+    'help': 'Displays this Onionr help menu',
+    'version': 'Displays the Onionr version',
+    'config': 'Configures something and adds it to the file',
+
+    'start': 'Starts the Onionr daemon',
+    'stop': 'Stops the Onionr daemon',
+
+    'stats': 'Displays node statistics',
+    'details': 'Displays the web password, public key, and human readable public key',
+
+    'enable-plugin': 'Enables and starts a plugin',
+    'disable-plugin': 'Disables and stops a plugin',
+    'reload-plugin': 'Reloads a plugin',
+    'create-plugin': 'Creates directory structure for a plugin',
+
+    'add-peer': 'Adds a peer to database',
+    'list-peers': 'Displays a list of peers',
+    'add-file': 'Create an Onionr block from a file',
+    'get-file': 'Get a file from Onionr blocks',
+    'import-blocks': 'import blocks from the disk (Onionr is transport-agnostic!)',
+    'listconn': 'list connected peers',
+    'pex': 'exchange addresses with peers (done automatically)',
+    'blacklist-block': 'deletes a block by hash and permanently removes it from your node',
+    'introduce': 'Introduce your node to the public Onionr network',
+    'friend': '[add|remove] [public key/id]',
+    'add-id': 'Generate a new ID (key pair)',
+    'change-id': 'Change active ID',
+    'open-home': 'Open your node\'s home/info screen'
+        }

onionr/onionrcommands/daemonlaunch.py

@@ -0,0 +1,142 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    launch the api server and communicator
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+import os, time, sys, platform, sqlite3
+from threading import Thread
+import onionr, api, logger, communicator
+import onionrevents as events
+from netcontroller import NetController
+def daemon(o_inst):
+    '''
+        Starts the Onionr communication daemon
+    '''
+
+    # remove runcheck if it exists
+    if os.path.isfile('data/.runcheck'):
+        logger.debug('Runcheck file found on daemon start, deleting in advance.')
+        os.remove('data/.runcheck')
+
+    Thread(target=api.API, args=(o_inst, o_inst.debug, onionr.API_VERSION)).start()
+    Thread(target=api.PublicAPI, args=[o_inst.getClientApi()]).start()
+    try:
+        time.sleep(0)
+    except KeyboardInterrupt:
+        logger.debug('Got keyboard interrupt, shutting down...')
+        time.sleep(1)
+        o_inst.onionrUtils.localCommand('shutdown')
+
+    apiHost = ''
+    while apiHost == '':
+        try:
+            with open(o_inst.onionrCore.publicApiHostFile, 'r') as hostFile:
+                apiHost = hostFile.read()
+        except FileNotFoundError:
+            pass
+        time.sleep(0.5)
+    onionr.Onionr.setupConfig('data/', self = o_inst)
+
+    if o_inst._developmentMode:
+        logger.warn('DEVELOPMENT MODE ENABLED (NOT RECOMMENDED)', timestamp = False)
+    net = NetController(o_inst.onionrCore.config.get('client.public.port', 59497), apiServerIP=apiHost)
+    logger.debug('Tor is starting...')
+    if not net.startTor():
+        o_inst.onionrUtils.localCommand('shutdown')
+        sys.exit(1)
+    if len(net.myID) > 0 and o_inst.onionrCore.config.get('general.security_level') == 0:
+        logger.debug('Started .onion service: %s' % (logger.colors.underline + net.myID))
+    else:
+        logger.debug('.onion service disabled')
+    logger.debug('Using public key: %s' % (logger.colors.underline + o_inst.onionrCore._crypto.pubKey))
+    time.sleep(1)
+
+    o_inst.onionrCore.torPort = net.socksPort
+    communicatorThread = Thread(target=communicator.startCommunicator, args=(o_inst, str(net.socksPort)))
+    communicatorThread.start()
+    
+    while o_inst.communicatorInst is None:
+        time.sleep(0.1)
+
+    # print nice header thing :)
+    if o_inst.onionrCore.config.get('general.display_header', True):
+        o_inst.header()
+
+    # print out debug info
+    o_inst.version(verbosity = 5, function = logger.debug)
+    logger.debug('Python version %s' % platform.python_version())
+
+    logger.debug('Started communicator.')
+
+    events.event('daemon_start', onionr = o_inst)
+    try:
+        while True:
+            time.sleep(3)
+            # Debug to print out used FDs (regular and net)
+            #proc = psutil.Process()
+            #print('api-files:',proc.open_files(), len(psutil.net_connections()))
+            # Break if communicator process ends, so we don't have left over processes
+            if o_inst.communicatorInst.shutdown:
+                break
+            if o_inst.killed:
+                break # Break out if sigterm for clean exit
+    except KeyboardInterrupt:
+        pass
+    finally:
+        o_inst.onionrCore.daemonQueueAdd('shutdown')
+        o_inst.onionrUtils.localCommand('shutdown')
+    net.killTor()
+    time.sleep(3)
+    o_inst.deleteRunFiles()
+    return
+
+def kill_daemon(o_inst):
+    '''
+        Shutdown the Onionr daemon
+    '''
+
+    logger.warn('Stopping the running daemon...', timestamp = False)
+    try:
+        events.event('daemon_stop', onionr = o_inst)
+        net = NetController(o_inst.onionrCore.config.get('client.port', 59496))
+        try:
+            o_inst.onionrCore.daemonQueueAdd('shutdown')
+        except sqlite3.OperationalError:
+            pass
+
+        net.killTor()
+    except Exception as e:
+        logger.error('Failed to shutdown daemon.', error = e, timestamp = False)
+    return
+
+def start(o_inst, input = False, override = False):
+    if os.path.exists('.onionr-lock') and not override:
+        logger.fatal('Cannot start. Daemon is already running, or it did not exit cleanly.\n(if you are sure that there is not a daemon running, delete .onionr-lock & try again).')
+    else:
+        if not o_inst.debug and not o_inst._developmentMode:
+            lockFile = open('.onionr-lock', 'w')
+            lockFile.write('')
+            lockFile.close()
+        o_inst.running = True
+        o_inst.daemon()
+        o_inst.running = False
+        if not o_inst.debug and not o_inst._developmentMode:
+            try:
+                os.remove('.onionr-lock')
+            except FileNotFoundError:
+                pass

onionr/onionrcommands/filecommands.py

@@ -0,0 +1,49 @@
+import base64, sys, os
+import logger
+from onionrblockapi import Block
+def add_file(o_inst, singleBlock=False, blockType='bin'):
+    '''
+        Adds a file to the onionr network
+    '''
+
+    if len(sys.argv) >= 3:
+        filename = sys.argv[2]
+        contents = None
+
+        if not os.path.exists(filename):
+            logger.error('That file does not exist. Improper path (specify full path)?')
+            return
+        logger.info('Adding file... this might take a long time.')
+        try:
+            with open(filename, 'rb') as singleFile:
+                blockhash = o_inst.onionrCore.insertBlock(base64.b64encode(singleFile.read()), header=blockType)
+            if len(blockhash) > 0:
+                logger.info('File %s saved in block %s' % (filename, blockhash))
+        except:
+            logger.error('Failed to save file in block.', timestamp = False)
+    else:
+        logger.error('%s add-file <filename>' % sys.argv[0], timestamp = False)
+
+def getFile(o_inst):
+    '''
+        Get a file from onionr blocks
+    '''
+    try:
+        fileName = sys.argv[2]
+        bHash = sys.argv[3]
+    except IndexError:
+        logger.error("Syntax %s %s" % (sys.argv[0], '/path/to/filename <blockhash>'))
+    else:
+        logger.info(fileName)
+
+        contents = None
+        if os.path.exists(fileName):
+            logger.error("File already exists")
+            return
+        if not o_inst.onionrUtils.validateHash(bHash):
+            logger.error('Block hash is invalid')
+            return
+
+        with open(fileName, 'wb') as myFile:
+            myFile.write(base64.b64decode(Block(bHash, core=o_inst.onionrCore).bcontent))
+    return

onionr/onionrcommands/keyadders.py

@@ -0,0 +1,49 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    add keys (transport and pubkey)
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sys
+import logger
+def add_peer(o_inst):
+    try:
+        newPeer = sys.argv[2]
+    except IndexError:
+        pass
+    else:
+        if o_inst.onionrUtils.hasKey(newPeer):
+            logger.info('We already have that key')
+            return
+        logger.info("Adding peer: " + logger.colors.underline + newPeer)
+        try:
+            if o_inst.onionrCore.addPeer(newPeer):
+                logger.info('Successfully added key')
+        except AssertionError:
+            logger.error('Failed to add key')
+
+def add_address(o_inst):
+    try:
+        newAddress = sys.argv[2]
+        newAddress = newAddress.replace('http:', '').replace('/', '')
+    except IndexError:
+        pass
+    else:
+        logger.info("Adding address: " + logger.colors.underline + newAddress)
+        if o_inst.onionrCore.addAddress(newAddress):
+            logger.info("Successfully added address.")
+        else:
+            logger.warn("Unable to add address.")

onionr/onionrcommands/onionrstatistics.py

@@ -0,0 +1,110 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This module defines commands to show stats/details about the local node
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import os, uuid, time
+import logger, onionrutils
+from onionrblockapi import Block
+import onionr
+
+def show_stats(o_inst):
+    try:
+        # define stats messages here
+        totalBlocks = len(o_inst.onionrCore.getBlockList())
+        signedBlocks = len(Block.getBlocks(signed = True))
+        messages = {
+            # info about local client
+            'Onionr Daemon Status' : ((logger.colors.fg.green + 'Online') if o_inst.onionrUtils.isCommunicatorRunning(timeout = 9) else logger.colors.fg.red + 'Offline'),
+
+            # file and folder size stats
+            'div1' : True, # this creates a solid line across the screen, a div
+            'Total Block Size' : onionrutils.humanSize(onionrutils.size(o_inst.dataDir + 'blocks/')),
+            'Total Plugin Size' : onionrutils.humanSize(onionrutils.size(o_inst.dataDir + 'plugins/')),
+            'Log File Size' : onionrutils.humanSize(onionrutils.size(o_inst.dataDir + 'output.log')),
+
+            # count stats
+            'div2' : True,
+            'Known Peers' : str(len(o_inst.onionrCore.listPeers()) - 1),
+            'Enabled Plugins' : str(len(o_inst.onionrCore.config.get('plugins.enabled', list()))) + ' / ' + str(len(os.listdir(o_inst.dataDir + 'plugins/'))),
+            'Stored Blocks' : str(totalBlocks),
+            'Percent Blocks Signed' : str(round(100 * signedBlocks / max(totalBlocks, 1), 2)) + '%'
+        }
+
+        # color configuration
+        colors = {
+            'title' : logger.colors.bold,
+            'key' : logger.colors.fg.lightgreen,
+            'val' : logger.colors.fg.green,
+            'border' : logger.colors.fg.lightblue,
+
+            'reset' : logger.colors.reset
+        }
+
+        # pre-processing
+        maxlength = 0
+        width = o_inst.getConsoleWidth()
+        for key, val in messages.items():
+            if not (type(val) is bool and val is True):
+                maxlength = max(len(key), maxlength)
+        prewidth = maxlength + len(' | ')
+        groupsize = width - prewidth - len('[+] ')
+
+        # generate stats table
+        logger.info(colors['title'] + 'Onionr v%s Statistics' % onionr.ONIONR_VERSION + colors['reset'])
+        logger.info(colors['border'] + '-' * (maxlength + 1) + '+' + colors['reset'])
+        for key, val in messages.items():
+            if not (type(val) is bool and val is True):
+                val = [str(val)[i:i + groupsize] for i in range(0, len(str(val)), groupsize)]
+
+                logger.info(colors['key'] + str(key).rjust(maxlength) + colors['reset'] + colors['border'] + ' | ' + colors['reset'] + colors['val'] + str(val.pop(0)) + colors['reset'])
+
+                for value in val:
+                    logger.info(' ' * maxlength + colors['border'] + ' | ' + colors['reset'] + colors['val'] + str(value) + colors['reset'])
+            else:
+                logger.info(colors['border'] + '-' * (maxlength + 1) + '+' + colors['reset'])
+        logger.info(colors['border'] + '-' * (maxlength + 1) + '+' + colors['reset'])
+    except Exception as e:
+        logger.error('Failed to generate statistics table.', error = e, timestamp = False)
+
+def show_details(o_inst):
+    details = {
+        'Node Address' : o_inst.get_hostname(),
+        'Web Password' : o_inst.getWebPassword(),
+        'Public Key' : o_inst.onionrCore._crypto.pubKey,
+        'Human-readable Public Key' : o_inst.onionrCore._utils.getHumanReadableID()
+    }
+
+    for detail in details:
+        logger.info('%s%s: \n%s%s\n' % (logger.colors.fg.lightgreen, detail, logger.colors.fg.green, details[detail]), sensitive = True)
+
+def show_peers(o_inst):
+    randID = str(uuid.uuid4())
+    o_inst.onionrCore.daemonQueueAdd('connectedPeers', responseID=randID)
+    while True:
+        try:
+            time.sleep(3)
+            peers = o_inst.onionrCore.daemonQueueGetResponse(randID)
+        except KeyboardInterrupt:
+            break
+        if not type(peers) is None:
+            if peers not in ('', 'failure', None):
+                if peers != False:
+                    logger.info('Peers: %s' % peers)
+                else:
+                    logger.warn('Daemon probably not running. Unable to list connected peers.')
+                break

onionr/onionrcommands/plugincommands.py

@@ -0,0 +1,88 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    plugin CLI commands
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+import sys
+import logger, onionrplugins as plugins
+
+def enable_plugin(o_inst):
+    if len(sys.argv) >= 3:
+        plugin_name = sys.argv[2]
+        logger.info('Enabling plugin "%s"...' % plugin_name)
+        plugins.enable(plugin_name, o_inst)
+    else:
+        logger.info('%s %s <plugin>' % (sys.argv[0], sys.argv[1]))
+
+def disable_plugin(o_inst):
+
+    if len(sys.argv) >= 3:
+        plugin_name = sys.argv[2]
+        logger.info('Disabling plugin "%s"...' % plugin_name)
+        plugins.disable(plugin_name, o_inst)
+    else:
+        logger.info('%s %s <plugin>' % (sys.argv[0], sys.argv[1]))
+
+def reload_plugin(o_inst):
+    '''
+        Reloads (stops and starts) all plugins, or the given plugin
+    '''
+
+    if len(sys.argv) >= 3:
+        plugin_name = sys.argv[2]
+        logger.info('Reloading plugin "%s"...' % plugin_name)
+        plugins.stop(plugin_name, o_inst)
+        plugins.start(plugin_name, o_inst)
+    else:
+        logger.info('Reloading all plugins...')
+        plugins.reload(o_inst)
+
+
+def create_plugin(o_inst):
+    '''
+        Creates the directory structure for a plugin name
+    '''
+
+    if len(sys.argv) >= 3:
+        try:
+            plugin_name = re.sub('[^0-9a-zA-Z_]+', '', str(sys.argv[2]).lower())
+
+            if not plugins.exists(plugin_name):
+                logger.info('Creating plugin "%s"...' % plugin_name)
+
+                os.makedirs(plugins.get_plugins_folder(plugin_name))
+                with open(plugins.get_plugins_folder(plugin_name) + '/main.py', 'a') as main:
+                    contents = ''
+                    with open('static-data/default_plugin.py', 'rb') as file:
+                        contents = file.read().decode()
+
+                    # TODO: Fix $user. os.getlogin() is   B U G G Y
+                    main.write(contents.replace('$user', 'some random developer').replace('$date', datetime.datetime.now().strftime('%Y-%m-%d')).replace('$name', plugin_name))
+
+                with open(plugins.get_plugins_folder(plugin_name) + '/info.json', 'a') as main:
+                    main.write(json.dumps({'author' : 'anonymous', 'description' : 'the default description of the plugin', 'version' : '1.0'}))
+
+                logger.info('Enabling plugin "%s"...' % plugin_name)
+                plugins.enable(plugin_name, o_inst)
+            else:
+                logger.warn('Cannot create plugin directory structure; plugin "%s" exists.' % plugin_name)
+
+        except Exception as e:
+            logger.error('Failed to create plugin directory structure.', e)
+    else:
+        logger.info('%s %s <plugin>' % (sys.argv[0], sys.argv[1]))

onionr/onionrcommands/pubkeymanager.py

@@ -0,0 +1,101 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This module defines ID-related CLI commands
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+import sys, getpass
+import logger
+from onionrusers import onionrusers
+def add_ID(o_inst):
+    try:
+        sys.argv[2]
+        assert sys.argv[2] == 'true'
+    except (IndexError, AssertionError) as e:
+        newID = o_inst.onionrCore._crypto.keyManager.addKey()[0]
+    else:
+        logger.warn('Deterministic keys require random and long passphrases.')
+        logger.warn('If a good passphrase is not used, your key can be easily stolen.')
+        logger.warn('You should use a series of hard to guess words, see this for reference: https://www.xkcd.com/936/')
+        pass1 = getpass.getpass(prompt='Enter at least %s characters: ' % (o_inst.onionrCore._crypto.deterministicRequirement,))
+        pass2 = getpass.getpass(prompt='Confirm entry: ')
+        if o_inst.onionrCore._crypto.safeCompare(pass1, pass2):
+            try:
+                logger.info('Generating deterministic key. This can take a while.')
+                newID, privKey = o_inst.onionrCore._crypto.generateDeterministic(pass1)
+            except onionrexceptions.PasswordStrengthError:
+                logger.error('Must use at least 25 characters.')
+                sys.exit(1)
+        else:
+            logger.error('Passwords do not match.')
+            sys.exit(1)
+        o_inst.onionrCore._crypto.keyManager.addKey(pubKey=newID, 
+        privKey=privKey)
+    logger.info('Added ID: %s' % (o_inst.onionrUtils.bytesToStr(newID),))
+
+def change_ID(o_inst):
+    try:
+        key = sys.argv[2]
+    except IndexError:
+        logger.error('Specify pubkey to use')
+    else:
+        if o_inst.onionrUtils.validatePubKey(key):
+            if key in o_inst.onionrCore._crypto.keyManager.getPubkeyList():
+                o_inst.onionrCore.config.set('general.public_key', key)
+                o_inst.onionrCore.config.save()
+                logger.info('Set active key to: %s' % (key,))
+                logger.info('Restart Onionr if it is running.')
+            else:
+                logger.error('That key does not exist')
+        else:
+            logger.error('Invalid key %s' % (key,))
+
+def friend_command(o_inst):
+    friend = ''
+    try:
+        # Get the friend command
+        action = sys.argv[2]
+    except IndexError:
+        logger.info('Syntax: friend add/remove/list [address]')
+    else:
+        action = action.lower()
+        if action == 'list':
+            # List out peers marked as our friend
+            for friend in onionrusers.OnionrUser.list_friends(o_inst.onionrCore):
+                logger.info(friend.publicKey + ' - ' + friend.getName())
+        elif action in ('add', 'remove'):
+            try:
+                friend = sys.argv[3]
+                if not o_inst.onionrUtils.validatePubKey(friend):
+                    raise onionrexceptions.InvalidPubkey('Public key is invalid')
+                if friend not in o_inst.onionrCore.listPeers():
+                    raise onionrexceptions.KeyNotKnown
+                friend = onionrusers.OnionrUser(o_inst.onionrCore, friend)
+            except IndexError:
+                logger.error('Friend ID is required.')
+            except onionrexceptions.KeyNotKnown:
+                o_inst.onionrCore.addPeer(friend)
+                friend = onionrusers.OnionrUser(o_inst.onionrCore, friend)
+            finally:
+                if action == 'add':
+                    friend.setTrust(1)
+                    logger.info('Added %s as friend.' % (friend.publicKey,))
+                else:
+                    friend.setTrust(0)
+                    logger.info('Removed %s as friend.' % (friend.publicKey,))
+        else:
+            logger.info('Syntax: friend add/remove/list [address]')

onionr/onionrcrypto.py

@@ -0,0 +1,299 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This file handles Onionr's cryptography.
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import nacl.signing, nacl.encoding, nacl.public, nacl.hash, nacl.pwhash, nacl.utils, nacl.secret, os, binascii, base64, hashlib, logger, onionrproofs, time, math, sys, hmac
+import onionrexceptions, keymanager, core
+# secrets module was added into standard lib in 3.6+
+if sys.version_info[0] == 3 and sys.version_info[1] < 6:
+    from dependencies import secrets
+elif sys.version_info[0] == 3 and sys.version_info[1] >= 6:
+    import secrets
+import config
+
+class OnionrCrypto:
+    def __init__(self, coreInstance):
+        config.reload()
+        self._core = coreInstance
+        self._keyFile = self._core.dataDir + 'keys.txt'
+        self.pubKey = None
+        self.privKey = None
+        self.secrets = secrets
+        self.deterministicRequirement = 25 # Min deterministic password/phrase length
+        self.HASH_ID_ROUNDS = 2000
+        self.keyManager = keymanager.KeyManager(self)
+
+        # Load our own pub/priv Ed25519 keys, gen & save them if they don't exist
+        if os.path.exists(self._keyFile):
+            if len(config.get('general.public_key', '')) > 0:
+                self.pubKey = config.get('general.public_key')
+            else:
+                self.pubKey = self.keyManager.getPubkeyList()[0]
+            self.privKey = self.keyManager.getPrivkey(self.pubKey)
+        else:
+            keys = self.generatePubKey()
+            self.pubKey = keys[0]
+            self.privKey = keys[1]
+            self.keyManager.addKey(self.pubKey, self.privKey)
+        return
+
+    def edVerify(self, data, key, sig, encodedData=True):
+        '''Verify signed data (combined in nacl) to an ed25519 key'''
+        try:
+            key = nacl.signing.VerifyKey(key=key, encoder=nacl.encoding.Base32Encoder)
+        except nacl.exceptions.ValueError:
+            #logger.debug('Signature by unknown key (cannot reverse hash)')
+            return False
+        except binascii.Error:
+            logger.warn('Could not load key for verification, invalid padding')
+            return False
+        retData = False
+        sig = base64.b64decode(sig)
+        try:
+            data = data.encode()
+        except AttributeError:
+            pass
+        if encodedData:
+            try:
+                retData = key.verify(data, sig) # .encode() is not the same as nacl.encoding
+            except nacl.exceptions.BadSignatureError:
+                pass
+        else:
+            try:
+                retData = key.verify(data, sig)
+            except nacl.exceptions.BadSignatureError:
+                pass
+        return retData
+
+    def edSign(self, data, key, encodeResult=False):
+        '''Ed25519 sign data'''
+        try:
+            data = data.encode()
+        except AttributeError:
+            pass
+        key = nacl.signing.SigningKey(seed=key, encoder=nacl.encoding.Base32Encoder)
+        retData = ''
+        if encodeResult:
+            retData = key.sign(data, encoder=nacl.encoding.Base64Encoder).signature.decode() # .encode() is not the same as nacl.encoding
+        else:
+            retData = key.sign(data).signature
+        return retData
+
+    def pubKeyEncrypt(self, data, pubkey, encodedData=False):
+        '''Encrypt to a public key (Curve25519, taken from base32 Ed25519 pubkey)'''
+        retVal = ''
+        box = None
+        data = self._core._utils.strToBytes(data)
+        
+        pubkey = nacl.signing.VerifyKey(pubkey, encoder=nacl.encoding.Base32Encoder()).to_curve25519_public_key()
+
+        if encodedData:
+            encoding = nacl.encoding.Base64Encoder
+        else:
+            encoding = nacl.encoding.RawEncoder
+        
+        box = nacl.public.SealedBox(pubkey)
+        retVal = box.encrypt(data, encoder=encoding)
+
+        return retVal
+
+    def pubKeyDecrypt(self, data, pubkey='', privkey='', encodedData=False):
+        '''pubkey decrypt (Curve25519, taken from Ed25519 pubkey)'''
+        decrypted = False
+        if encodedData:
+            encoding = nacl.encoding.Base64Encoder
+        else:
+            encoding = nacl.encoding.RawEncoder
+        if privkey == '':
+            privkey = self.privKey
+        ownKey = nacl.signing.SigningKey(seed=privkey, encoder=nacl.encoding.Base32Encoder()).to_curve25519_private_key()
+
+        if self._core._utils.validatePubKey(privkey):
+            privkey = nacl.signing.SigningKey(seed=privkey, encoder=nacl.encoding.Base32Encoder()).to_curve25519_private_key()
+            anonBox = nacl.public.SealedBox(privkey)
+        else:
+            anonBox = nacl.public.SealedBox(ownKey)
+        decrypted = anonBox.decrypt(data, encoder=encoding)
+        return decrypted
+
+    def symmetricEncrypt(self, data, key, encodedKey=False, returnEncoded=True):
+        '''Encrypt data to a 32-byte key (Salsa20-Poly1305 MAC)'''
+        if encodedKey:
+            encoding = nacl.encoding.Base64Encoder
+        else:
+            encoding = nacl.encoding.RawEncoder
+
+        # Make sure data is bytes
+        if type(data) != bytes:
+            data = data.encode()
+
+        box = nacl.secret.SecretBox(key, encoder=encoding)
+
+        if returnEncoded:
+            encoding = nacl.encoding.Base64Encoder
+        else:
+            encoding = nacl.encoding.RawEncoder
+
+        encrypted = box.encrypt(data, encoder=encoding)
+        return encrypted
+
+    def symmetricDecrypt(self, data, key, encodedKey=False, encodedMessage=False, returnEncoded=False):
+        '''Decrypt data to a 32-byte key (Salsa20-Poly1305 MAC)'''
+        if encodedKey:
+            encoding = nacl.encoding.Base64Encoder
+        else:
+            encoding = nacl.encoding.RawEncoder
+        box = nacl.secret.SecretBox(key, encoder=encoding)
+
+        if encodedMessage:
+            encoding = nacl.encoding.Base64Encoder
+        else:
+            encoding = nacl.encoding.RawEncoder
+        decrypted = box.decrypt(data, encoder=encoding)
+        if returnEncoded:
+            decrypted = base64.b64encode(decrypted)
+        return decrypted
+
+    def generateSymmetric(self):
+        '''Generate a symmetric key (bytes) and return it'''
+        return binascii.hexlify(nacl.utils.random(nacl.secret.SecretBox.KEY_SIZE))
+
+    def generatePubKey(self):
+        '''Generate a Ed25519 public key pair, return tuple of base32encoded pubkey, privkey'''
+        private_key = nacl.signing.SigningKey.generate()
+        public_key = private_key.verify_key.encode(encoder=nacl.encoding.Base32Encoder())
+        return (public_key.decode(), private_key.encode(encoder=nacl.encoding.Base32Encoder()).decode())
+
+    def generateDeterministic(self, passphrase, bypassCheck=False):
+        '''Generate a Ed25519 public key pair from a password'''
+        passStrength = self.deterministicRequirement
+        passphrase = self._core._utils.strToBytes(passphrase) # Convert to bytes if not already
+        # Validate passphrase length
+        if not bypassCheck:
+            if len(passphrase) < passStrength:
+                raise onionrexceptions.PasswordStrengthError("Passphase must be at least %s characters" % (passStrength,))
+        # KDF values
+        kdf = nacl.pwhash.argon2id.kdf
+        salt = b"U81Q7llrQcdTP0Ux" # Does not need to be unique or secret, but must be 16 bytes
+        ops = nacl.pwhash.argon2id.OPSLIMIT_SENSITIVE
+        mem = nacl.pwhash.argon2id.MEMLIMIT_SENSITIVE
+
+        key = kdf(32, passphrase, salt, opslimit=ops, memlimit=mem) # Generate seed for ed25519 key
+        key = nacl.signing.SigningKey(key)
+        return (key.verify_key.encode(nacl.encoding.Base32Encoder).decode(), key.encode(nacl.encoding.Base32Encoder).decode())
+
+    def pubKeyHashID(self, pubkey=''):
+        '''Accept a ed25519 public key, return a truncated result of X many sha3_256 hash rounds'''
+        if pubkey == '':
+            pubkey = self.pubKey
+        prev = ''
+        pubkey = pubkey.encode()
+        for i in range(self.HASH_ID_ROUNDS):
+            try:
+                prev = prev.encode()
+            except AttributeError:
+                pass
+            hasher = hashlib.sha3_256()
+            hasher.update(pubkey + prev)
+            prev = hasher.hexdigest()
+        result = prev
+        return result
+
+    def sha3Hash(self, data):
+        try:
+            data = data.encode()
+        except AttributeError:
+            pass
+        hasher = hashlib.sha3_256()
+        hasher.update(data)
+        return hasher.hexdigest()
+
+    def blake2bHash(self, data):
+        try:
+            data = data.encode()
+        except AttributeError:
+            pass
+        return nacl.hash.blake2b(data)
+
+    def verifyPow(self, blockContent):
+        '''
+            Verifies the proof of work associated with a block
+        '''
+        retData = False
+
+        dataLen = len(blockContent)
+
+        try:
+            blockContent = blockContent.encode()
+        except AttributeError:
+            pass
+
+        blockHash = self.sha3Hash(blockContent)
+        try:
+            blockHash = blockHash.decode() # bytes on some versions for some reason
+        except AttributeError:
+            pass
+        
+        difficulty = onionrproofs.getDifficultyForNewBlock(blockContent, ourBlock=False)
+        
+        if difficulty < int(config.get('general.minimum_block_pow')):
+            difficulty = int(config.get('general.minimum_block_pow'))
+        mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode()
+        puzzle = mainHash[:difficulty]
+
+        if blockHash[:difficulty] == puzzle:
+            # logger.debug('Validated block pow')
+            retData = True
+        else:
+            logger.debug("Invalid token, bad proof")
+
+        return retData
+
+    @staticmethod
+    def replayTimestampValidation(timestamp):
+        if core.Core()._utils.getEpoch() - int(timestamp) > 2419200:
+            return False
+        else:
+            return True
+
+    @staticmethod
+    def safeCompare(one, two):
+        # Do encode here to avoid spawning core
+        try:
+            one = one.encode()
+        except AttributeError:
+            pass
+        try:
+            two = two.encode()
+        except AttributeError:
+            pass
+        return hmac.compare_digest(one, two)
+        
+    @staticmethod
+    def randomShuffle(theList):
+        myList = list(theList)
+        shuffledList = []
+        myListLength = len(myList) + 1
+        while myListLength > 0:
+            removed = secrets.randbelow(myListLength)
+            try:
+                shuffledList.append(myList.pop(removed))
+            except IndexError:
+                pass
+            myListLength = len(myList)
+        return shuffledList

onionr/onionrevents.py

@@ -0,0 +1,72 @@
+'''
+    Onionr - P2P Microblogging Platform & Social network
+
+    This file deals with configuration management.
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+import config, logger, onionrplugins as plugins, onionrpluginapi as pluginapi
+from threading import Thread
+
+def get_pluginapi(onionr, data):
+    return pluginapi.pluginapi(onionr, data)
+
+def __event_caller(event_name, data = {}, onionr = None):
+    '''
+        DO NOT call this function, this is for threading code only.
+        Instead, call onionrevents.event
+    '''
+    for plugin in plugins.get_enabled_plugins():
+        try:
+            call(plugins.get_plugin(plugin), event_name, data, get_pluginapi(onionr, data))
+        except ModuleNotFoundError as e:
+            logger.warn('Disabling nonexistant plugin "%s"...' % plugin)
+            plugins.disable(plugin, onionr, stop_event = False)
+        except Exception as e:
+            logger.warn('Event "%s" failed for plugin "%s".' % (event_name, plugin))
+            logger.debug(str(e))
+
+
+def event(event_name, data = {}, onionr = None, threaded = True):
+    '''
+        Calls an event on all plugins (if defined)
+    '''
+
+    if threaded:
+        thread = Thread(target = __event_caller, args = (event_name, data, onionr))
+        thread.start()
+        return thread
+    else:
+        __event_caller(event_name, data, onionr)
+
+def call(plugin, event_name, data = None, pluginapi = None):
+    '''
+        Calls an event on a plugin if one is defined
+    '''
+
+    if not plugin is None:
+        try:
+            attribute = 'on_' + str(event_name).lower()
+
+            if hasattr(plugin, attribute):
+                #logger.debug('Calling event ' + str(event_name))
+                getattr(plugin, attribute)(pluginapi, data)
+
+            return True
+        except Exception as e:
+            return False
+    else:
+        return True

onionr/onionrexceptions.py

@@ -0,0 +1,97 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This file contains exceptions for onionr
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+# general exceptions
+class NotFound(Exception):
+    pass
+class Unknown(Exception):
+    pass
+class Invalid(Exception):
+    pass
+
+# communicator exceptions
+class OnlinePeerNeeded(Exception):
+    pass
+
+# crypto exceptions
+class InvalidPubkey(Exception):
+    pass
+
+class KeyNotKnown(Exception):
+    pass
+
+class DecryptionError(Exception):
+    pass
+
+class PasswordStrengthError(Exception):
+    pass
+
+# block exceptions
+
+class ReplayAttack(Exception):
+    pass
+
+class DifficultyTooLarge(Exception):
+    pass
+
+class InvalidMetadata(Exception):
+    pass
+
+class BlacklistedBlock(Exception):
+    pass
+
+class DataExists(Exception):
+    pass
+
+class NoDataAvailable(Exception):
+    pass
+
+class InvalidHexHash(Exception):
+    '''When a string is not a valid hex string of appropriate length for a hash value'''
+    pass
+
+class InvalidProof(Exception):
+    '''When a proof is invalid or inadequate'''
+    pass
+
+# network level exceptions
+class MissingPort(Exception):
+    pass
+
+class InvalidAddress(Exception):
+    pass
+
+class InvalidAPIVersion(Exception):
+    pass
+
+# file exceptions
+
+class DiskAllocationReached(Exception):
+    pass
+
+# onionrsocket exceptions
+
+class MissingAddress(Exception):
+    pass
+
+# Contact exceptions
+
+class ContactDeleted(Exception):
+    pass

onionr/onionrfragment/__init__.py

@@ -0,0 +1,73 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This file contains the OnionrFragment class which implements the fragment system
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+# onionr:10ch+10ch+10chgdecryptionkey
+import core, sys, binascii, os
+
+FRAGMENT_SIZE = 0.25
+TRUNCATE_LENGTH = 30
+
+class OnionrFragment:
+    def __init__(self, uri=None):
+        uri = uri.replace('onionr:', '')
+        count = 0
+        blocks = []
+        appendData = ''
+        key = ''
+        for x in uri:
+            if x == 'k':
+                key = uri[uri.index('k') + 1:]
+            appendData += x
+            if count == TRUNCATE_LENGTH:
+                blocks.append(appendData)
+                appendData = ''
+                count = 0
+            count += 1
+        self.key = key
+        self.blocks = blocks
+        return
+
+    @staticmethod
+    def generateFragments(data=None, coreInst=None):
+        if coreInst is None:
+            coreInst = core.Core()
+
+        key = os.urandom(32)
+        data = coreInst._crypto.symmetricEncrypt(data, key).decode()
+        blocks = []
+        blockData = b""
+        uri = "onionr:"
+        total = sys.getsizeof(data)
+        for x in data:
+            blockData += x.encode()
+            if round(len(blockData) / len(data), 3) > FRAGMENT_SIZE:
+                blocks.append(core.Core().insertBlock(blockData))
+                blockData = b""
+
+        for bl in blocks:
+            uri += bl[:TRUNCATE_LENGTH]
+        uri += "k"
+        uri += binascii.hexlify(key).decode()
+        return (uri, key)
+
+if __name__ == '__main__':
+    uri = OnionrFragment.generateFragments("test")[0]
+    print(uri)
+    OnionrFragment(uri)