Removed old scripts

+ Added SIGUSR1 handler to dump stacktrace of all active threads

.dockerignore

@@ -1,4 +1,6 @@
 onionr/data/**/*
 onionr/data
-RUN-WINDOWS.bat
 MY-RUN.sh
+Dockerfile
+.dockerignore
+.git

.env

@@ -0,0 +1 @@
+PYTHONPATH=./venv/bin/python310:./src/

.github/stale.yml

@@ -0,0 +1,7 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 0
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 0
+# Label to use when marking an issue as stale
+staleLabel: wontfix
+closeComment: false

.github/workflows/greetings.yml

@@ -0,0 +1,13 @@
+name: Greetings
+
+on: [pull_request, issues]
+
+jobs:
+  greeting:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/first-interaction@v1
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        issue-message: 'Message that will be displayed on users'' first issue'
+        pr-message: 'Hi, please make the PR at https://gitlab.com/beardog/Onionr'' first pr'

.gitignore

@@ -1,32 +1,44 @@
 __pycache__/
-onionr/data/config.ini
-onionr/data/*.db
-onionr/data-old/*
-onionr/data*
-onionr/testdata
-onionr/*.pyc
-onionr/*.log
-onionr/data/hs/hostname
-onionr/data/*
-onionr/data-backup/*
-onionr/gnupg/*
+src/data/config.ini
+src/data/*.db
+src/data-old/*
+src/data*
+src/tor
+src/tor.exe
+src/testdata
+src/*.pyc
+src/*.log
+src/data/hs/hostname
+src/data/*
+src/data-backup/*
 run.sh
-onionr/data-encrypted.dat
-onionr/.onionr-lock
-core
-.vscode/*
-venv/*
-onionr/fs*
+src/.onionr-lock
+daemon-true.txt
+.vscode/tags
+.vscode/settings.json
+
+venv*
+src/fs*
+src/tmp/*
+testdata/*
+*.dll
+*.exe
+
+.mypy_cache/
+
+dist/*
 
 # log files
 output.log
 *.log
-onionr/output.log
-onionr/*.log
-onionr/data/output.log
-onionr/data/*.log
+src/output.log
+src/*.log
+src/data/output.log
+src/data/*.log
 
 # package files
 onionr-*.pkg.tar.gz
 pkg/
-src/
+spawnnodes.py
+
+static-data/tor-node-list.dat

.gitlab-ci.yml

@@ -1,7 +1,11 @@
 test:
-  image: ubuntu:bionic
+  image: python
   script:
   - apt-get update -qy
-  - apt-get install -y python3-pip tor
-  - pip3 install -r requirements.txt
+  - apt-get install -y tor
+  - apt-get install -y firefox-esr
+  - pip3 install --require-hashes -r requirements.txt
+  - pip3 install --require-hashes -r requirements-dev.txt
+  - pip3 install --require-hashes -r requirements-notifications.txt
+  - python3 --version
   - make test

.onionr.yml

@@ -0,0 +1,57 @@
+# lazypkg config
+
+name: onionr-git
+version: 0.1
+release: 1
+summary: anonymous P2P communication platform
+description: Onionr is a decentralized, peer-to-peer communication network, designed
+  to be anonymous and resistant to (meta)data analysis, spam, and corruption.
+license: GPL
+website: https://onionr.net/
+contact: contact@onionr.net
+maintainer: Aaron Esau
+maintainer-contact: aur@aaronesau.com
+
+relationships:
+- conflicts: onionr2
+
+sources:
+- git: https://gitlab.com/beardog/onionr.git
+  branch: master
+
+dependencies:
+- deb: git
+  required: true
+- deb: curl
+  required: true
+- deb: tor
+  required: true
+- deb: python3.7
+  pkgbuild: python
+  build: true
+  required: true
+- deb: python3-setuptools
+  pkgbuild: python-setuptools
+  build: true
+  required: true
+- deb: python3-pip
+  pkgbuild: python-pip
+  build: true
+  required: true
+
+movements:
+- install/onionr: /usr/bin/
+  chown: root:root
+  chmod: 755
+- install/onionr.service: /etc/systemd/system/
+  chown: root:root
+  chmod: 644
+- '.': /usr/share/onionr
+  chown: root:root
+  chmod: 755
+
+scripts:
+- build: install/build.sh
+- pre_install: install/pre_install.sh
+- post_install: install/post_install.sh
+

.vscode/launch.json

@@ -0,0 +1,23 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Onionr: init",
+            "type": "python",
+            "request": "launch",
+            "program": "${workspaceFolder}/src/__init__.py",
+            "cwd": "${workspaceFolder}/src/",
+            "args" : ["start"]
+        },
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "cwd": "${workspaceFolder}/src/"
+        }
+    ]
+}

.vscode/tasks.json

@@ -0,0 +1,58 @@
+{
+    // See https://go.microsoft.com/fwlink/?LinkId=733558
+    // for the documentation about the tasks.json format
+    "version": "2.0.0",
+    "tasks": [
+      {
+        "label": "Make new test",
+        "type": "shell",
+        "command": "cp tests/test_template.py tests/new-test.py",
+        "group": "test",
+        "presentation": {
+          "reveal": "always",
+          "panel": "new"
+        }
+      },
+      {
+        "label": "Run tests",
+        "type": "shell",
+        "command": "source venv/bin/activate; make test",
+        "group": "test",
+        "presentation": {
+          "reveal": "always",
+          "panel": "new"
+        }
+      },
+      {
+        "label": "Run test by name",
+        "type": "shell",
+        "command": "source venv/bin/activate; scripts/run-unit-test-by-name.py",
+        "group": "test",
+        "presentation": {
+          "reveal": "always",
+          "panel": "new"
+        }
+      },
+      {
+        "label": "Enable dev config",
+        "type": "process",
+        "command": "scripts/enable-dev-config.py",
+        "group": "dev",
+        "problemMatcher": []
+      },
+      {
+        "label": "Disable dev config",
+        "type": "process",
+        "command": "scripts/disable-dev-config.py",
+        "group": "dev",
+        "problemMatcher": []
+      },
+      {
+        "label": "Format default config",
+        "type": "process",
+        "command": "scripts/pretty-default-config.py",
+        "group": "dev",
+        "problemMatcher": []
+      }
+    ]
+  }

CHANGELOG.md

@@ -0,0 +1,48 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [8.0.1] - 2020-12-22
+
+* Fix subprocess in 3.9x with dumb hack
+* Dependency bumps
+
+## [8.0.0] - 2020-12-04
+
+* Decrease PoW until better implementation is made
+
+
+## [7.2.0] - 2020-12-03
+
+* Purge blocks not meeting current pow on startup
+* Check block POW before LAN sync
+* WSL fixes
+
+## [7.1.0] - 2020-11-23
+
+* Check for ownership of existing dirs in createdirs, this prevents the rare edge case where a user might use a home directory in a location an attacker could write (allowing arbitrary code execution via plugins). This was already partially mitigated by the chmod of the home directory in any case, but this further fixes the issue.
+
+## [7.0.0] - 2020-11-22
+
+* Removed communicator timers
+* Removed direct connections and chat (these will be either plugins or separate programs/processes in the future)
+
+
+## [5.1.0] - 2020-09-07
+
+* Moved plugin web files to be in the plugin folder to reduce staticfiles blueprint coupling
+* Added basic sidebar on index page
+* Many bug fixes
+
+
+## [5.0.1] - 2020-08-08
+
+* bumped deadsimplekv to 0.3.2
+* bumped urllib3 to 1.25.10
+
+## [5.0.0] - 2020-07-23
+
+- Removed single-process POW support (was only needed on Windows)

CONTRIBUTING.md

@@ -12,18 +12,11 @@ Contributors in project-related spaces/contexts are expected to follow the [Code
 
 Bugs can be reported using GitLab issues. Please try to see if an issue is already opened for a particular thing.
 
-Please provide the following information when reporting a bug:
-
-* Operating system
-* Python version
-* Onionr version
-* Onionr logs or output with errors, any possible relevant information.
-* A description of what you were doing before the bug was experienced
-* Screenshots can often be helpful, but videos are rarely helpful.
+See the [issue template](ISSUE_TEMPLATE.md) for what to include when reporting a bug.
 
 If a bug is a security issue, please contact us privately.
 
-And most importantly, please be patient. Onionr is a free open source project maintained by volunteers in their free time.
+Please be patient. Onionr is a free open source project maintained by volunteers in their free time.
 
 ## Asking Questions
 
@@ -35,6 +28,6 @@ Do your best to use good english.
 
 For any non-trivial changes, please get in touch with us first to discuss your plans.
 
-Please try to use a similar coding style as the project.
+Please try to use a similar coding style as the project. We like PEP-8 despite being lazy with it in the past.
 
 **Thanks for contributing to Onionr!**

Dockerfile

@@ -1,28 +1,31 @@
-FROM ubuntu:bionic
+FROM python:3.10
+EXPOSE 8080
 
-#Base settings
-ENV HOME /root
+USER root
+
+RUN mkdir /app
+WORKDIR /app
+
+ENV ONIONR_DOCKER=true
 
 #Install needed packages
-RUN apt update && apt install -y python3 python3-dev python3-pip tor locales nano sqlite3
+RUN apt-get update && apt-get install -y tor locales
 
 RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
     locale-gen
-ENV LANG en_US.UTF-8  
-ENV LANGUAGE en_US:en  
-ENV LC_ALL en_US.UTF-8  
+ENV LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8
 
-WORKDIR /srv/
-ADD ./requirements.txt /srv/requirements.txt
-RUN pip3 install -r requirements.txt
+ADD ./requirements-x86-all-plugins.txt /app/requirements.txt
+RUN pip3 install --require-hashes -r requirements.txt
 
-WORKDIR /root/
 #Add Onionr source
-COPY . /root/
-VOLUME /root/data/
+COPY . /app/
 
-#Set upstart command
-CMD bash
+VOLUME /app/data/
 
-#Expose ports
-EXPOSE 8080
+#Default to running as nonprivileged user
+RUN chmod g=u -R /app
+USER 1000
+ENV HOME=/app
+
+CMD ["bash", "./onionr.sh", "start"]

ISSUE_TEMPLATE.md

@@ -5,8 +5,9 @@
 # Steps to Reproduce
 
 # Version Information
-Onionr: 
-OS: 
-Python: 
-Tor: 
-I2P: 
+
+* Onionr: 
+* OS: 
+* Python: 
+* Tor: 
+* CPU:

LICENSE.txt

@@ -1,6 +1,3 @@
-The Onionr logo was created by [Anhar Ismail](https://github.com/anharismail) under the [Creative Commons Attribution 4.0 International License](https://creativecommons.org/licenses/by/4.0/). 
-
-
                     GNU GENERAL PUBLIC LICENSE
                        Version 3, 29 June 2007
 

Makefile

@@ -25,16 +25,16 @@ test:
 
 soft-reset:
 	@echo "Soft-resetting Onionr..."
-	rm -f onionr/$(ONIONR_HOME)/blocks/*.dat onionr/data/*.db onionr/$(ONIONR_HOME)/block-nonces.dat | true > /dev/null 2>&1
+	./onionr.sh soft-reset
 	@./onionr.sh version | grep -v "Failed" --color=always
 
 reset:
 	@echo "Hard-resetting Onionr..."
-	rm -rf onionr/$(ONIONR_HOME)/ | true > /dev/null 2>&1
+	rm -rf $(ONIONR_HOME)/ | true > /dev/null 2>&1
 	cd onionr/static-data/www/ui/; rm -rf ./dist; python compile.py
 	#@./onionr.sh version | grep -v "Failed" --color=always
 
 plugins-reset:
 	@echo "Resetting plugins..."
-	rm -rf onionr/$(ONIONR_HOME)/plugins/ | true > /dev/null 2>&1
+	@./onionr.sh reset-plugins | true > /dev/null 2>&1
 	@./onionr.sh version | grep -v "Failed" --color=always

PKGBUILD

@@ -1,20 +0,0 @@
-url="https://onionr.net/"
-pkgname="onionr"
-pkgver=0.0
-pkgrel=1
-pkgdesc="P2P anonymous storage network"
-arch=("x86_64")
-license=('GPL')
-source=("onionr-${pkgver}::git+https://gitlab.com/beardog/onionr.git#branch=master")
-md5sums=('SKIP')
-makedepends=('git', 'python3-pip')
-
-build() {
-        cd "$pkgname-${pkgver}"
-        make
-}
-
-package() {
-        cd "$pkgname-${pkgver}"
-        # make install
-}

README.md

@@ -5,92 +5,106 @@
 </p>
 
 <p align="center">
-    Private P2P Communication Network 🕵️
+    Privacy Respecting Communication Network 📡
+</p>
+<p align="center">
+    WIP anonymous social platform, mail, file sharing and marketplace
 </p>
 
-(***pre-alpha & experimental, not well tested or easy to use yet***)
+<img src="https://img.shields.io/badge/License-aGPLv3-yellow"> <img src='https://img.shields.io/badge/python%20version%20%F0%9F%90%8D-3.10+-blue'>
 
-[![Open Source Love](https://badges.frapsoft.com/os/v3/open-source.png?v=103)](https://github.com/ellerbrock/open-source-badges/)
-<img src='https://gitlab.com/beardog/Onionr/badges/master/build.svg'> - [Onionr.net](https://onionr.net/) - [.onion](http://onionr.onionkvc5ibm37bmxwr56bdxcdnb6w3wm4bdghh5qo6f6za7gn7styid.onion/)
+<a href='https://twitter.com/onionrnet'><img src='https://img.shields.io/twitter/follow/onionrnet?style=social'></a> - [Discord](https://discord.gg/DVF2bEAzrt) - Matrix: #onionr:amorgan.xyz
 
-<hr>
+|                                      |                            |                            |
+| -----------                          | -----------                | -----------                |
+| [Install](#install-and-run-on-linux) | [Features](#main-features) | [Screenshots](#screenshots)|
+| [Docs](#documentation)               | [Get involved](#help-out)  | [Onionr.net](https://onionr.net/)/[.onion](http://onionrbak72t5zhbzuey2fdkpczlvhowgcpqc6uoyrd3uxztzxwz5cyd.onion/)                  |
 
-**The main repository for this software is at https://gitlab.com/beardog/Onionr/**
 
-# About
+---
 
-Onionr is a decentralized, peer-to-peer communication network, designed to be anonymous and resistant to (meta)data analysis, spam, and corruption.
+**The main repository for this software is at https://gitlab.com/beardog/onionr/**
 
-Onionr stores data in independent packages referred to as 'blocks'. The blocks are synced to all other nodes in the network. Blocks and user IDs cannot be easily proven to have been created by a particular user. Even if there is enough evidence to believe that a specific user created a block, nodes still operate behind Tor or I2P and as such cannot be trivially unmasked.
+***Note that this README reflects the state of the rewrite, and not the original alpha network***
+
+Mirrors [Gitea](https://git.voidnet.tech/kev/onionr)
+
+
+Onionr ("Onion Relay") is a decentralized/distributed peer-to-peer communication network, designed to be anonymous and resistant to (meta)data analysis, spam, and corruption.
+
+Onionr gives the individual the ability to speak freely, without fear of surveillance and censorship.
+
+---
+
+# Onionr internals
+
+## Blocks
+
+At the core, Onionr is an anonymous Distributed Hash Table (DHT) syncing prepackaged blocks using a simple Gossip protocol with Dandelion++ as an overlay network on top of Tor and I2P.
+
+Onionr stores data in independent packages referred to as 'blocks'. The blocks are distributed to all nodes, but are not required to be stored. Blocks and user IDs cannot be easily proven to have been created by a particular user. Even if Dandelion++ is defeated and there is enough evidence to believe that a specific node is linked to a block's creation, nodes still operate behind Tor or I2P and as such cannot be trivially unmasked. Anonymity is achieved by a stateless network, with no given indication of what node a block originates from. In fact, since one is not required to participate in routing or storage to insert a message, blocks often do not originate from any identifiable node, similar to how Bitcoin transactions do not necessarily originate from a wallet directly associated with a node.
+
+## Onionr Gossip
+
+Onionr works via epidemic/gossip style routing, with message delivery taking roughly log<sub>C</sub>(N) cycles where C is the number of nodes to send a message to each cycle and N is the number of connected nodes. So a network of 100 million nodes can deliver messages in a few minutes even with high packet loss and malfunctioning nodes.
+
+Through Dandelion++ message forwarding and key privacy, it is intended to be nigh impossible to discover the identity of a message creator or recipient. Via long-term traffic analysis, a well funded adversary may discover the most probable node(s) to be creating a set of related blocks, however doing so would only lead them to a node behind Tor. As the first node that a block appears on is almost always not the creator of the block, there is plausible deniability regarding the true creator of the block.
 
 Users are identified by ed25519/curve25519 public keys, which can be used to sign blocks or send encrypted data.
 
 Onionr can be used for mail, as a social network, instant messenger, file sharing software, or for encrypted group discussion.
 
-The whitepaper (subject to change prior to alpha release) is available [here](docs/whitepaper.md).
+Since Onionr is technically just a data format, any routing scheme can  technically be used to pass messages.
 
-![Tor stinks slide image](docs/tor-stinks-02.png)
+The whitepaper is available [here](docs/whitepaper.md).
 
-## Main Features
+---
+
+# Main Features
 
 * [X] 🌐 Fully p2p/decentralized, no trackers or other single points of failure
-* [X] 🔒 End to end encryption of user data
-* [X] 📢 Optional non-encrypted blocks, useful for blog posts or public file sharing
-* [X] 💻 Easy HTTP API for integration to websites
 * [X] 🕵️ Metadata analysis resistance and anonymity
 * [X] 📡 Transport agnosticism (no internet required)
 
-## Software Suite
 
-Onionr ships with various application plugins ready for use out of the box:
+# Roadmap
 
-Currently usable:
-
-* Mail
-* Public anonymous chat
-* Simple webpage hosting - Will be greatly extended
-* File sharing (Work in progress)
-
-Not yet usable:
-
-* Instant messaging
-* Forum/BBS
-
-**Onionr API and functionality is subject to non-backwards compatible change during pre-alpha development**
-
-# Screenshots
-
-<img alt='Node statistics page screenshot' src='docs/onionr-1.png' width=600>
-
-Node statistics
-
-<img alt='Friend/contact manager screenshot' src='docs/onionr-2.png' width=600>
-
-Friend/contact manager
-
-<img alt='Encrypted, metadata-masking mail application screenshot' src='docs/onionr-3.png' width=600>
-
-Encrypted, metadata-masking mail application. One of the first distributed mail systems to have basic forward secrecy.
+See [ROADMAP.md](ROADMAP.md)
 
 # Documentation
 
 More docs coming soon.
 
-* [Block specification](docs/specs/block-spec.md)
-* [HTTP API](docs/http-api.md)
 
 # Install and Run on Linux
 
-The following applies to Ubuntu Bionic. Other distros may have different package or command names.
+The following applies to Ubuntu Bionic. Other distributions may have different package or command names.
+
+Master may be unstable, you should use the latest release tag. (checkout via git: `$ git checkout release-latest`)
 
 `$ sudo apt install python3-pip python3-dev tor`
 
-* Have python3.6+, python3-pip, Tor (daemon, not browser) installed. python3-dev is recommended.
-* Clone the git repo: `$ git clone https://gitlab.com/beardog/onionr`
+* Have python3.10, python3-pip, Tor (daemon, not browser) installed. python3-dev is recommended.
+* You may need build-essentials or the equivalent of your platform
+* Clone the git repo: `$ git clone https://gitlab.com/beardog/onionr --tags`
 * cd into install direction: `$ cd onionr/`
-* Install the Python dependencies ([virtualenv strongly recommended](https://virtualenv.pypa.io/en/stable/userguide/)): `$ pip3 install --require-hashes -r requirements.txt`
+* Install the Python dependencies ([virtualenv strongly recommended](https://virtualenv.pypa.io/en/stable/userguide/)): `$ pip3 install --require-hashes -r requirements-x86-all-plugins.txt`
 
-(--require-hashes is intended to prevent exploitation via compromise of PyPi/CA certificates)
+Require-hashes is suggested for supply-chain security but is optional. The hashes are not correct for ARM machines. If you are just running a node or want a bare-bones install you can use requirements-base-x86.txt and selectively install the requirements.txt files in static-data/official-plugins/ subdirectories
+
+## Run Onionr
+
+* Run Onionr normally:  `$ ./onionr.sh start`
+* Run Onionr in background as daemon: `$ ./start-daemon.sh`
+* Gracefully stop Onionr from CLI `$ ./onionr.sh stop`
+
+
+# Contact/Community
+
+* Email: onionr [ at ] voidnet.tech
+* Twitter: [@onionrnet](https://twitter.com/onionrnet)
+* Matrix: #onionr:amorgan.xyz
+* Discord: https://discord.gg/DVF2bEAzrt (Discord is bad for freedom and privacy, this is only provided for convenience)
 
 # Help out
 
@@ -99,35 +113,45 @@ Everyone is welcome to contribute. Help is wanted for the following:
 * Development (Get in touch first)
     * Creation of a shared lib for use from other languages and faster proof-of-work
     * Android and IOS development
-    * Windows and Mac support (already partially supported, testers needed)
-    * General bug fixes and development of new features
+    * Mac support (testers needed)
+    * Bug fixes and development of new features
 * Testing
+* Translations/localizations
 * UI/UX design
 * Running stable nodes
 * Security review/audit
-* Automatic I2P setup
+* I2P support
+
+## Watch the talk from BSidesPDX 2019
+
+<a href="https://www.youtube.com/watch?v=mrULtmSkKxg">
+<img src="docs/talk.png" alt="improving anonymous networking talk link" width="600">
+</a>
 
 ## Contribute money:
 
-Donating at least $5 gets you cool Onionr stickers. Get in touch if you want them.
+Donating at least $3 gets you cool Onionr stickers. Get in touch if you want them.
 
-Bitcoin: [1onion55FXzm6h8KQw3zFw2igpHcV7LPq](bitcoin:1onion55FXzm6h8KQw3zFw2igpHcV7LPq) (Contact us for a unique address or for other coins)
+![sticker](docs/sticker.png)
 
-Monero: 4B5BA24d1P3R5aWEpkGY5TP7buJJcn2aSGBVRQCHhpiahxeB4aWsu15XwmuTjC6VF62NApZeJGTS248RMVECP8aW73Uj2ax
 
-USD (Card/Paypal): [Ko-Fi](https://www.ko-fi.com/beardogkf)
+* Bitcoin: [bc1qpayme9rlpkch0qp3r79lvm5racr7t6llauwfmg](bitcoin:bc1qpayme9rlpkch0qp3r79lvm5racr7t6llauwfmg) (Contact us for a unique address or for other coins)
 
-Note: probably not tax deductible
+* Monero: 4B5BA24d1P3R5aWEpkGY5TP7buJJcn2aSGBVRQCHhpiahxeB4aWsu15XwmuTjC6VF62NApZeJGTS248RMVECP8aW73Uj2ax
 
-# Contact
+* USD (Card/Paypal (no account required)): [Ko-Fi](https://www.ko-fi.com/beardogkf)
 
-Email: beardog [ at ] mailbox.org
+* Sign up for [privacy.com (refferal link)](https://privacy.com/join/FNNDF) to protect your personal information when contributing or shopping elsewhere, we both get $5 USD.
 
-Onionr Mail: TRH763JURNY47QPBTTQ4LLPYCYQK6Q5YA33R6GANKZK5C5DKCIGQ
+Note: not tax deductible
 
-## Disclaimers and legal
+# Security
 
-No matter how good Onionr and other software gets, there will always be ways for clever or well-funded adversaries to break your security. 
+Onionr is alpha software. This means it is unstable, probably insecure, and experimental.
+
+No matter how good Onionr and other software gets, there will always be ways for clever or well-funded adversaries to break your security.
+
+Onionr does not protect your identity if you associate your user ID with your name either on Onionr or elsewhere.
 
 *Do not rely on Onionr or any other software to hold up if your life or liberty are at stake.*
 
@@ -139,12 +163,8 @@ The Tor Project and I2P developers do not own, create, or endorse this project,
 
 Tor is a trademark for the Tor Project. We do not own it.
 
-The 'open source badge' is by Maik Ellerbrock and is licensed under a Creative Commons Attribution 4.0 International License.
-
 ## Onionr Logo
 
-The Onionr logo was created by [Anhar Ismail](https://github.com/anharismail) under the [Creative Commons Attribution 4.0 International License](https://creativecommons.org/licenses/by/4.0/). 
+The Onionr logo was created by [Anhar Ismail](https://github.com/anharismail) under the [Creative Commons Attribution 4.0 International License](https://creativecommons.org/licenses/by/4.0/).
 
 If you modify and redistribute our code ("forking"), please use a different logo and project name to avoid confusion. Please do not use the project name or logo in a way that makes it seem like we endorse you without our permission.
-
-![node web illustration](docs/onionr-web.png)

ROADMAP.md

@@ -0,0 +1,46 @@
+# Onionr Roadmap
+
+
+## Taraxacum Release (9.0)
+
+
+* [ X ] Implement new block format with verifiable delay function
+
+* [ X ] Implement overhauled gossip system with dandelion++
+
+    The new system is separated from the underlying networks used and makes it much easier to implement new transport methods. Dandelion++ is also better than the old block mixing we did.
+
+
+## Misc
+
+* [  ] Spruce up documentation
+* [  ] Restore LAN transport
+* [  ] Restore webUI as a plugin
+* [  ] Restore static site/file sharing plugin
+* [  ] Restore/reimplement mail plugin
+* [  ] Restore/reimplement friends plugin
+* [  ] Refresh test suite
+* [  ] Revamped key/encrypted messaging (encrypted blocks)
+
+
+
+## Web of trust release (~10.0)
+
+To facilitate the below market plugin/application, Onionr will need a web of trust system.
+
+* [ ] Web of trust plugin or module
+
+
+## Market Plugin Release (~10.1)
+
+The Onionr team believes the Monero community is currently lacking a good p2p market, and as such we hope to build a solution using Onionr as a transport. This may be a separate project and as opposed to a plugin.
+
+
+* A new marketplace plugin will be developed with the following *tenative* features:
+
+* [ ] Account management
+* [ ] Monero management
+* [ ] Store front discovery and search
+* [ ] Product listing search
+* [ ] User reviews
+

SECURITY.md

@@ -17,7 +17,7 @@ The following exploits are of particular interest:
   * Discovering true server IPs when behind Tor/I2P (aside from Tor/i2p-level attacks)
   * Easily discovering which nodes are the block creator
 * XSS, CSRF, clickjacking, DNS rebinding
-* Timing attacks against the local http server ([see blog post](https://www.chaoswebs.net/blog/timebleed-breaking-privacy-with-a-simple-timing-attack.html))
+* Timing attacks against the local http server that are exploitable from a browser ([see blog post](https://www.chaoswebs.net/blog/timebleed-breaking-privacy-with-a-simple-timing-attack.html))
 * Discovering direct connection servers as a non participant.
 * Cryptography/protocol issues
 * Denying nodes access to the network by segmenting them out with Sybil nodes

base-requirements.in

@@ -0,0 +1,8 @@
+PyNaCl==1.5.0
+psutil==5.9.1
+filenuke==0.0.0
+ujson==5.4.0
+cffi==1.15.1
+onionrblocks==7.0.0
+ordered-set==4.1.0
+json-rpc==1.13.0

docs/README.md

@@ -1,4 +1,6 @@
-# Onionr Documentation
+<img src="onionr-logo.png" width="200px">
+
+# Documentation
 
 The Onionr [whitepaper](whitepaper.md) is the best place to start both for users and developers.
 
@@ -16,4 +18,4 @@ The Onionr [whitepaper](whitepaper.md) is the best place to start both for users
 * [Technical overview](dev/overview.md)
 * [Project layout](dev/layout.md)
 * [Plugin development guide](dev/plugins.md)
-* [Testing](dev/testing.md)
+* [Auto generated API docs (HTML)](html/onionr/index.html)

docs/dev/daemon-events.md

@@ -0,0 +1,9 @@
+# DaemonEvents
+
+For things that need to be processed by the daemon
+
+Observer pattern
+
+Register listeners dynamically per event
+
+Spawn new greenlets

docs/dev/error-codes.md

@@ -0,0 +1,4 @@
+# Exit codes
+
+10: run with no command
+3: command does not exist

docs/dev/generating-requirements.txt.md

@@ -0,0 +1,19 @@
+# Generating requirements.txt
+
+To generate a requirements.txt file, install pip-tools from pip
+
+Onionr requirements files should have hashes to prevent backdooring by the pypi server.
+
+Put your package versions in requirements.in like normal. Child dependencies are usually not necessary:
+
+```
+requests==0.1.1
+flask==0.1.1
+```
+
+Then generate the requirements.txt:
+
+`$ pip-compile requirements.in --generate-hashes -o requirements.txt`
+
+
+Your requirements.txt will have hash-pinned requirements of all dependencies and child dependencies.

docs/dev/http-api.md

@@ -1,118 +0,0 @@
-# Onionr HTTP API
-
-All HTTP interfaces in the Onionr reference client use the [Flask](http://flask.pocoo.org/) web framework with the [gevent](http://www.gevent.org/) WSGI server.
-
-## Client & Public difference
-
-The client API server is a locked down interface intended for authenticated local communication. 
-
-The public API server is available only remotely from Tor & I2P. It is the interface in which peers use to communicate with one another.
-
-# Client API
-
-Please note: endpoints that simply provide static web app files are not documented here.
-
-* /serviceactive/pubkey
-    - Methods: GET
-    - Returns true or false based on if a given public key has an active direct connection service.
-* /queueResponseAdd/key (DEPRECATED)
-    - Methods: POST
-    - Accepts form key 'data' to set queue response information from a plugin
-    - Returns success if no error occurs
-* /queueResponse/key (DEPRECATED)
-    - Methods: GET
-    - Returns the queue response for a key. Returns failure with a 404 code if a code is not set.
-* /ping
-    - Methods: GET
-    - Returns "pong!"
-* /getblocksbytype/type
-    - Methods: GET
-    - Returns a list of stored blocks by a given type
-* /getblockbody/hash
-    - Methods: GET
-    - Returns the main data section of a block
-* /getblockdata/hash
-    - Methods: GET
-    - Returns the entire data contents of a block, including metadata.
-* /getblockheader/hash
-    - Methods: GET
-    - Returns the header (metadata section) of a block.
-* /hitcount
-    - Methods: GET
-    - Return the amount of requests the public api server has received this session
-* /lastconnect
-    - Methods: GET
-    - Returns the epoch timestamp of when the last incoming connection to the public API server was logged
-* /site/hash
-    - Methods: GET
-    - Returns HTML content out of a block
-* /waitforshare/hash
-    - Methods: POST
-    - Prevents the public API server from listing or sharing a block until it has been uploaded to at least 1 peer.
-* /shutdown
-    - Methods: GET
-    - Shutdown Onionr. You should probably use /shutdownclean instead.
-* /shutdownclean
-    - Methods: GET
-    - Tells the communicator daemon to shutdown Onionr. Slower but cleaner.
-* /getstats
-    - Methods: GET
-    - Returns some JSON serialized statistics
-* /getuptime
-    - Methods: GET
-    - Returns uptime in seconds
-* /getActivePubkey
-    - Methods: GET
-    - Returns the current active public key in base32 format
-* /getHumanReadable/pubkey
-    - Methods: GET
-    - Echos the specified public key in mnemonic format
-* /insertblock
-    - Methods: POST
-    - Accepts JSON data for creating a new block. 'message' contains the block data, 'to' specifies the peer's public key to encrypt the data to, 'sign' is a boolean for signing the message.
-
-# Public API
-
-v0
-
-* /
-    - Methods: GET
-    - Returns a basic HTML informational banner describing Onionr.
-* /getblocklist
-    - Methods: GET
-    - URI Parameters:
-        - date: unix epoch timestamp for offset
-    - Returns a list of block hashes stored on the node since an offset (all blocks if no timestamp is specified)
-* /getdata/block-hash
-    - Methods: GET
-    - Returns data for a block based on a provided hash
-* /www/file-path
-    - Methods: GET
-    - Returns file data. Intended for manually sharing file data directly from an Onionr node.
-* /ping
-    - Methods: GET
-    - Returns 'pong!'
-* /pex
-    - Methods: GET
-    - Returns a list of peer addresses reached within recent time
-* /announce
-    - Methods: POST
-    - Accepts form data for 'node' (valid node address) and 'random' which is a nonce when hashed (blake2b_256) in the format `hash(peerAddress+serverAddress+nonce)`, begins with at least 5 zeros.
-    - Returns 200 with 'Success' if no error occurs. If the post is invalid, 'failure' with code 406 is returned.
-* /upload
-    - Methods: POST
-    - Accepts form data for 'block' as a 'file' upload.
-    - Returns 200 with 'success' if no error occurs. If the block cannot be accepted, 'failure' with 400 is returned.
-
-# Direct Connection API
-
-These are constant endpoints available on direct connection servers. Plugin endpoints for direct connections are not documented here.
-
-* /ping
-    - Methods: GET
-    - Returns 200 with 'pong!'
-
-* /close
-    - Methods: GET
-    - Kills the direct connection server, destroying the onion address.
-    - Returns 200 with 'goodbye'

docs/dev/running-tests.md

@@ -0,0 +1,44 @@
+# Running Onionr tests
+
+Onionr has four types of tests:
+
+* unittests
+* integration tests
+* selenium tests (web tests)
+* runtime-tests
+
+
+## unittests
+
+Onionr uses Python's built in unittest module. These tests are located in tests/ (top level)
+
+Run all tests with `$ make test`, which will also run integration tests.
+
+Please note that one unittest tests if runtime-tests have passed recently. This is simply a forceful reminder to run those tests as well.
+
+You can also run a single unittest in a loop by using the script scripts/run-unit-test-by-name.py
+
+## integration tests
+
+These tests are pretty basic and test on stdout of Onionr commands.
+
+They are also run from `$ make test`
+
+The runtime-tests do most of the actual integration testing.
+
+## selenium tests
+
+These are browser automation tests to test if the UI is working as how it should for a user.
+
+There's only a couple and they're incomplete, so they can be ignored for now (test manually)
+
+## runtime-tests
+
+These are important. They look into the Onionr client Flask app when Onionr daemon is running and test a bunch of things.
+
+If you do it a lot you should make your own Onionr network (disable official bootstrap)
+
+You run this while the daemon is running (probably should make sure onboarding is done), with `$ onionr.sh runtime-test`
+
+It's necessary to do this before running `$ make test` for unittesting
+

docs/dev/security-mechanisms.md

@@ -0,0 +1,43 @@
+# Onionr Security Mechanisms
+
+
+## bigbrother 👁️
+
+Bigbrother is a cheeky module that uses Python3.8+ sys auditing events to log and/or block certain sensitive events.
+
+It has a little overhead, so one can disable it in config in general.security_auditing
+
+[ChaosWebs.net/blog/preventing-arbitrary-code-execution-in-python38-with-auditing.html](https://chaoswebs.net/blog/preventing-arbitrary-code-execution-in-python38-with-auditing.html)
+
+### Threat model
+
+It is intended to log bugs leaking private file system information, block+log network leaks, and block+log eval-like arbitrary code execution. It is not intended to block malicious browser scripts or malicious Python plugins. It cannot work with subprocesses that do not activate the module.
+
+It's not intended to be bulletproof by any means, but it helps.
+
+### What big brother does
+
+* Disk access checks for disk access outside. Only logs, does not block
+* Network leaks. (Non Tor/LAN) Blocks and logs
+* Arbitrary code execution: logs and blocks non-whitelisted bytecode importing/compiling and subprocesses.
+
+
+## Sybil attacks
+
+As with any decentralized network, sybil nodes could collude to spy or cause mayhem. Due to the gossip nature of Onionr, sybil nodes would have a hard time fully stopping the network. In terms of spying, they could not conclusively prove the origin of messages due to the multiple transport nature of the network and layering behind Tor/etc.
+
+## Tor configuration
+
+When managed by Onionr, Tor has a control port password that gets stored in Onionr config.
+
+Tor is also configured to reject requests to non-onion services, which helps to stop redirect based denial of service attacks.
+
+## Web security
+
+Onionr secures both it's main web APIs with anti-dns-rebinding logic, which validates the host header used in connections to it. This is to prevent exfiltration of data and side channel deanonymization.
+
+Onionr secures the client API with a token that must be passed in most requests, with the exception of static API files. This is to prevent CSRF and side channel deanonymization.
+
+Onionr binds most services to random loopback addresses to reduce all cross-site web attacks, including discovery of Onionr on a computer from a normal website. This is not supported on Mac because Mac does not support non 'typical' loopback addresses.
+
+Onionr has a strict content-security-policy, rejecting all non-localhost requests and denying inline scripts and similar insecure sources.

docs/dev/selected-papers.md

@@ -0,0 +1,9 @@
+# Interesting papers related to Onionr development
+
+A paper being listed here is not end-all-be-all endorsement of every detail inside.
+
+* [Epidemic Routing for Partially-Connected Ad Hoc Networks](https://web.archive.org/web/20200208074703/http://issg.cs.duke.edu/epidemic/epidemic.pdf)
+* [Freenet: A distibuted decentralized information storage and retrieval system](https://freenetproject.org/assets/papers/ddisrs.pdf)
+* [Protecting Free Expression Online with Freenet](https://freenetproject.org/assets/papers/ddisrs.pdf)
+* [Bitmessage: A Peer‐to‐Peer Message Authentication and Delivery System](https://archive.org/details/BitmessageWhitepaper/)
+* [MuON: Epidemic based Mutual Anonymity](https://web.archive.org/web/20060901153544/http://www.csl.mtu.edu/cs6461/www/Reading/MuON_ICNP2005.pdf)

docs/dev/setup.md

@@ -0,0 +1,72 @@
+<h1 align="center">Onionr Developer Guide</h1>
+
+This page assumes that Onionr is already installed and normal user requirements are setup.
+
+The Onionr development environment is simple. All one really needs is a supported Python version (currently 3.7-3.8 as of writing).
+
+There are additional requirements specified in requirements-dev.txt
+
+**Developers agree to the [CoC](../../CODE_OF_CONDUCT.md) and to contribute new code under GPLv3 or later**. Developers should stick to PEP8 in most cases, and write unittests or integration tests where possible.
+
+## Developer Scripts
+
+run-onionr-node.py can be used to start a node with specific parameters
+
+Intended to be used from VSCode (but could work otherwise), there are scripts in scripts/ named enable/disable-dev-config.py.
+These make modifications to the default config for the purpose of making testing Onionr nodes easier.
+Be sure to disable it again before pushing work.
+
+There are also scripts to generate new tests.
+
+*When adjusting PoW, it will make your node not compatible with the existing network*
+
+Generally, one should disable bootstrap list usage when testing non trivial changes. This is a config option: general.use_bootstrap_list. and can be configured through enable-dev-config.py and run-onionr-node.py
+
+
+# Current state of Onionr [2021-01-14]
+
+Onionr in it's current form is functional, albeit buggy.
+
+
+## Current major components
+
+Onionr runs via two main HTTP gevent servers serving Flask apps.
+
+Dir: apiservers
+
+* 1 Parent app hosts all public API endpoints for the Tor transport.
+* 1 Parent app hosts all UI-related files and endpoints. Some commands and internal modules interact with this API as well
+* The HTTP servers have strict anti-dns-rebinding and CSRF countermeasures, so there is a script to craft requests to the UI-related API in scripts/
+* Block storage is currently handled via metadata in sqlite (mostly defunct now), and block data storage in a different database. This is in blocks/ in running Onionr daemon data directory
+* cryptography is currently handled in onionrcrypto/ except for ephemeral messages which are handled by onionr
+* Transport clients run from looping threads mostly created in communicator/__init__.py, this includes block lookups and uploading on the Tor transport
+
+## Road map
+
+There are several big ways Onionr will be improved in the next major version:
+
+* Migration to the [new modular block system](https://git.voidnet.tech/kev/onionrblocks)
+    * Probability proof of work -> verifiable delay function
+    * Friend system built on top of signing proofs (Private networks?)
+* Gossip transport improvements such as with neighbor improvements. See streamfill/ and [simple gossip](https://github.com/onion-sudo/simplegossip) for incomplete experiments
+
+* Finish removing "communicator"
+* I2P transports
+    * Gossip
+    * Torrents (patch for sha1?)
+* Modular transports
+    * Currently transports are just threads coupled together.
+    * It would be better if there was a generic way to tell any loaded transport what blocks are wanted and feed back received blocks to the database
+* Migrate to SafeDB for peers and blocks
+    * SafeDB wrapper that contacts http endpoint to store if it is running, otherwise directly open DB
+* Separate UI logic from daemon. Refactor code to
+* Improve cryptography
+    * Restore phrases or deterministic keys (generate key from password, be very careful)
+    * Change identities to be dual keys (ed25519+curve25519)
+    * Finish treasurechest
+        * Interact via [named pipes](https://en.wikipedia.org/wiki/Named_pipe)
+        * Ephemeral key management
+        * Encrypt/decrypt/sign/verify functions to keep key out of main memory
+        * PGP-like symmetric messages
+
+

docs/dev/specs/block-spec.md

@@ -1,4 +1,4 @@
-# Onionr Block Spec v1.1.0
+# Onionr Block Spec v2.0.0
 
 # Block Description
 
@@ -20,13 +20,13 @@ The metadata section has the following fields. If a block contains any other fie
 
 ## meta
 
-Max byte size: 1000
+Max byte size (when in escaped json string format): 1000
 
-Meta is a string field which can contain arbitrary sub fields. It is intended for applications and plugins to use it for arbitrary metadata information. In the reference client, if the data section is encrypted or signed, the meta section also is.
+Meta is a string field which can contain arbitrary sub fields. It is intended for applications and plugins to use it for arbitrary metadata information. If the data section is encrypted or signed, the meta section also is.
 
 Common meta fields, such as 'type' are used by the reference Onionr client to describe the type of a block.
 
-## sig
+## sig (optional)
 
 Max byte size: 200
 
@@ -34,30 +34,30 @@ Sig is a field for storing public key signatures of the block, typically ed25519
 
 Note: the max field size is larger than a EdDSA signature (which is what is typically used) in order to allow other primitives for signing in alternative implementations or future versions.
 
-## signer
+## signer (optional, necessary if sig is present)
 
 Max byte size: 200
 
 Signer is a field for specifying the public key which signed the block. In the reference client this is a base64 encoded ed25519 public key.
 
-## time
+## time (mandatory)
 
 Max byte size: 10
 
-Time is an integer field for specifying the time of which a block was created. The trustworthiness of this field is based on one's trust of the block creator, however blocks with a time field set in the future (past a reasonable clock skew) are thrown out by the reference client.
+Time is an integer field for specifying the time of which a block was created. The trustworthiness of this field is based on one's trust of the block creator, however blocks with a time field set in the future at the point of block receipt (past a reasonable clock skew) are thrown out by the reference client.
 
-## expire
+## expire (optional)
 
 Max byte size: 10
 
 Expire is an integer field for specifying the time of which the block creator has indicated that the block should be deleted. The purpose of this is for voluntarily freeing the burden of unwanted blocks on the Onionr network, rather than security/privacy (since blocks could be trivially kept past expiration). Regardless, the reference client deletes blocks after a preset time if the expire field is either not set or longer than the preset time.
 
-## pow
+## pow (effectively mandatory)
 
 Max byte size: 1000
 
 Pow is a field for placing the nonce found to make a block meet a target proof of work. In theory, a block could meet a target without a random token in this field.
 
-## encryptType
+## encryptType (optional)
 
-encryptType is a field to specify the mode of encryption for a block. The values supported by Onionr are 'asym' and 'sym'.
+encryptType is a field to specify the mode of encryption for a block. The values supported by Onionr are 'asym' and 'sym'.

docs/dev/specs/forward-secrecy.md

@@ -0,0 +1,5 @@
+# Onionr Forward Secrecy Spec v0.0.0
+
+# Introduction
+
+Due to the natural trade-offs of implementing [forward secrecy](https://en.wikipedia.org/wiki/Forward_secrecy) in a distributed, decentralized system, Onionr has optional ephemeral keys.

docs/docker.md

@@ -0,0 +1,28 @@
+# Running Onionr in Docker
+
+A Dockerfile is included in the root directory of Onionr.
+
+In Docker version 20.10 (and probably others), there is a strange bug where Onionr must be run with -it or stdout will be garbled and it may hang.
+
+## Clone and build the image
+
+`$ git clone https://git.voidnet.tech/kev/onionr/`
+`$ cd onionr`
+`$ sudo docker build -t onionr .`
+
+
+## Run Onionr
+
+`$ sudo docker run -it -p 8080:8080 onionr`
+
+Onionr will be accessible over any network interface by default, so make sure to either change the entry point bind-address argument or set a firewall rule.
+
+That said, Onionr does protect it's interface by default with a web token, which will be shown in stdout.
+
+**However, anyone who can access the port may be able to see what Onionr sites you have saved and potentially deanonymize your node**
+
+## View the UI
+
+Visit the address and port for the machine Onionr is running on, for example: http://192.168.1.5:8080/#<long-token-taken-from-stdout>
+
+If you want a secure connection to the interface, either use a proxy such as nginx or caddy, or use [SSH tunneling](./vps-cloud-guide.md).

docs/faq.md

@@ -0,0 +1,24 @@
+<h1 align="center">Onionr FAQ</h1>
+
+(Most of the FAQ answers apply to planned features and do not reflect in the demo network)
+
+* How does Onionr route messages?
+
+Onionr creates a message with an anti-spam proof (usually). It is then forwarded between nodes using binomial tree broadcast.
+
+The network is structured based on the particular transport being used, for Tor, .onion addresses are used to determine closeness between nodes.
+
+Churn due to connectivity changes or issues, membership changes or malicious activity means nodes need to sometimes look far away for missed messages.
+
+To help with scale, messages can be intelligently deleted or forwarded, for example a mail message can be marked by its recipient at a given point for deletion, or forum posts can be set to expire after a short period or after lack of activity.
+
+
+* Why do you use Python instead of [language]?
+
+I'm very comfortable in Python, and I believe it is a maintainable language if you use it correctly with modern features, also it has many quality libraries useful to the project, like pyncal and stem. Most places in the project are IO bound.
+
+
+* What do you think of bad actors who might use Onionr?
+
+Users should be able to exclude viewing, processing, and hosting content they do not want to. To this end, I want to enable user-configurable filtering based on lists provided by friends, fuzzy hashing, and voting.
+

docs/usage/deterministic.md

@@ -0,0 +1,7 @@
+# Deterministic address
+
+Deterministic addresses are simply Onionr user IDs that are generated from a passphrase rather than general entropy.
+
+To generate an Onionr deterministic, create a random passphrase of at least 25 characters of length. **It is very important that your passphrase is highly random. 9+ words is recommended for your passphrase**
+
+`$ onionr add-id true`

docs/usage/firststeps.md

@@ -2,6 +2,6 @@
 
 After installing Onionr, there are several things to do:
 
-1. Setup a [deterministic address](usage/deterministic.md) (optional)
-2. Add friends' ids
-3. Publish your id
+1. Setup a [deterministic address](deterministic.md) (optional)
+2. Start your node `/path/to/onionr.sh start`
+3. Copy your ID and share it with friends

docs/usage/install.md

@@ -1,13 +1,18 @@
 # Onionr Installation
 
-The following steps work broadly speaking for Windows, Mac, and Linux.
+The following steps work broadly speaking for WSL, Mac, and Linux.
 
-1. Verify python3.6+ is installed: if its not see https://www.python.org/downloads/
+1. Verify python3.7+ is installed: if not, see https://www.python.org/downloads/
 
 2. Verify Tor is installed (does not need to be running, binary can be put into system path or Onionr directory)
 
 3. [Optional but recommended]: setup virtual environment using [virtualenv](https://virtualenv.pypa.io/en/latest/), activate the virtual environment
 
-4. Clone Onionr: git clone https://gitlab.com/beardog/onionr
+4. Clone Onionr: `$ git clone https://gitlab.com/beardog/onionr`
 
-5. Install the Python module dependencies: pip3 install --require-hashes -r requirements.txt
+5. Install the Python module dependencies: `$ pip3 install --require-hashes -r requirements.txt`
+
+
+Note: if an alternative python install is needed, use virtualenv or run Onionr commands with:
+
+`$ /path/to/python /path/to/onionr/onionr/__init__.py`

docs/usage/mail.md

@@ -0,0 +1,16 @@
+To use OnionrMail, open the web interface:
+
+`$ onionr.sh openweb`
+
+Navigate to the mail interface from the navbar.
+
+## Receiving mail
+
+Copy your ID from the textbox below the navbar. Publish the ID so that someone (or general people) know your ID to send messages to it. Note that associating the ID with your real ID publicly removes anonymity.
+
+## Sending mail
+
+1. optionally add the recipient as a friend via the friends page.
+2. click compose in the mail menu
+3. either select the recipient from the friend list or paste their ID in manually into the recipient box
+4. send away!

docs/usage/pages.md

@@ -0,0 +1,59 @@
+Onionr sites come in two forms:
+
+* Single-page sites, identified by the hash of a single page contained within a single Onionr block.
+
+* Multi-page sites, identified by a user ID. Contains directory archives of a full site.
+
+
+# Metadata Awareness
+
+Before creating an Onionr site, one should be cautious of the metadata one could be leaking. For example, some HTML generators may insert author meta tags. Onionr does not filter out any web page data.
+
+# No JavaScript, no third-party resources
+
+Currently, in order to protect Onionr users, JavaScript is disabled within Onionr sites. JS will remain present in the HTML file, but be non functional. Additionally, third party resources outside of Onionr cannot be loaded.
+
+
+# Creating multi page sites
+
+Multi page sites are the most useful, as they can contain an arbitrary amount of static files.
+
+To create a single page site, create a directory for your site and write standard HTML file(s) within them. CSS, images and other files can be placed in the directory as well. The home page should be name index.html and in the parent level directory.
+
+Then, create a strong passphrase for the site. If the site will be updated, be sure to write it down or remember it. A strong passphrase can be generated by running:
+
+`$ scripts/passphrase-generator.py`
+
+Sample output: lovesick blubberer haemoglobin... and so on.
+
+## Generating or updating the site:
+
+`$ ./onionr.sh addsite`
+
+All files in the current working directory will be added to the site.
+
+The command will prompt for a passphrase.
+
+After the site is generated, a user ID that identifies the site will be outputted.
+
+# Creating single page sites
+
+Single page sites are incredibly straight forward.
+
+Single page sites cannot be modified or updated, but are somewhat more secure due to having lower complexity.
+
+To create a single page site, write a standard HTML file. Inline or data-uri CSS can be included, as well as data-uri images. Data-URI generators can be found online.
+
+After creating the HTML file, run this command:
+
+`$ ./onionr.sh addhtml filename.html`
+
+![single page screenshot](single-page.png)
+
+# Viewing sites
+
+To view a site, open the Onionr web interface and paste the site hash or ID into the site opener box that looks like this:
+
+![site opener box screenshot](site-opener.png)
+
+Then, press open.

docs/usage/safety.md

@@ -0,0 +1,6 @@
+Onionr and all privacy-oriented software are not silver bullets.
+
+For example, if one gives away too much about themselves on an Onionr forum, it doesn't matter how good Onionr's privacy is.
+
+
+TODO: expand this page

docs/vps-cloud-guide.md

@@ -0,0 +1,18 @@
+# Cloud/Server Hosting Onionr
+
+Cloud hosting is not the recommended way to run Onionr, as it gives the cloud provider control over your data.
+
+That said, it is quite useful and by running a 24/7 Onionr node you contribute to the network's health.
+
+![https://cock.li/img/cockbox.png](https://cockbox.org/?r=12)
+
+[Cockbox](https://cockbox.org/?r=12) is the recommended provider, as they do not demand personal information and accept Monero. This is an affiliate link, which gives us a commission at no expense or complication to you.
+
+
+1. [Install Onionr like normal](./basic-onionr-user-guide.pdf) or [from Docker](./docker.md)
+2. Run with specific host and port to bind to: `$ ./run-onionr-node.py --bind-address 0.0.0.0 --port 8080`
+3. Get the web security token: `$ ./onionr.sh get-web` (the long string at the end of the URL)
+4. [Configure Firefox for SSH tunneling](https://web.archive.org/web/20210123034529/https://gist.github.com/brentjanderson/6ed800376e53746d2d28ba7b6bdcdc12). `Set network.proxy.allow_hijacking_localhost` to true in about:config
+
+
+Disable random host binding in config (general.security.random_bind_ip = false) if you have Onionr auto start. You may also want to disable deniable block inserts.

docs/whitepaper.md

@@ -1,15 +1,18 @@
 <p align="center">
- <img src="onionr-logo.png" alt="<h1>Onionr</h1>" width=200>
+   <h1 align="center">Onionr</h1>
 </p>
+
 <p align="center">Anonymous, Decentralized, Distributed Network</p>
 
+June 2020
+
 # Introduction
 
 We believe that the ability to communicate freely with others is crucial for maintaining societal and personal liberty. The internet has provided humanity with the ability to spread information globally, but there are many persons and organizations who try to stifle the flow of information, sometimes with success.
 
-Internet censorship comes in many forms, state censorship, corporate consolidation of media, threats of violence, network exploitation (e.g. denial of service attacks) and other threats.
+Internet censorship comes in many forms, state censorship, threats of violence, network exploitation (e.g. denial of service attacks) and others.
 
-We hold that in order to protect individual privacy, users must have the ability to communicate anonymously and with decentralization. 
+We hold that in order to protect individual privacy, users must have the ability to communicate anonymously and with decentralization.
 
 We believe that in order to prevent censorship and loss of information, these measures must be in place:
 
@@ -57,12 +60,14 @@ To mitigate maliciously slow or unreliable nodes, Onionr builds a profile on nod
 
 ## Block Format
 
-Onionr blocks are very simple. They are structured in two main parts: a metadata section and a data section, with a line feed delimiting where metadata ends and data begins. 
+Onionr blocks are very simple. They are structured in two main parts: a metadata section and a data section, with a line feed delimiting where metadata ends and data begins.
 
 Metadata defines what kind of data is in a block, signature data, encryption settings, and other arbitrary information.
 
 Optionally, a random token can be inserted into the metadata for use in Proof of Work.
 
+The proof of work function should be a Verifiable Delay Function (VDF). We have chosen MiMC for it's simplicity.
+
 ### Block Encryption
 
 For encryption, Onionr uses ephemeral Curve25519 keys for key exchange and XSalsa20-Poly1305 as a symmetric cipher or optionally using only XSalsa20-Poly1305 with a pre-shared key.
@@ -83,9 +88,7 @@ Blocks are stored indefinitely until the allocated space is filled, at which poi
 
 ## Block Timestamping
 
-Onionr can provide evidence of when a block was inserted by requesting other users to sign a hash of the current time with the block data hash: sha3_256(time + sha3_256(block data)).
-
-This can be done either by the creator of the block prior to generation, or by any node after insertion.
+Onionr blocks are by default not accepted if their timestamp is set too far in the past, or is in the future.
 
 In addition, randomness beacons such as the one operated by [NIST](https://beacon.nist.gov/home), [Chile](https://beacon.clcert.cl/), or the hash of the latest blocks in a cryptocurrency network could be used to affirm that a block was at least not *created* before a given time.
 
@@ -97,7 +100,7 @@ The benefits of such a system are increased privacy, and the ability to anonymou
 
 # Threat Model
 
-The goal of Onionr is to provide a method of distributing information in a manner in which the difficulty of discovering the identity of those sending and receiving the information is greatly increased. In this section we detail what information we want to protect and who we're protecting it from. 
+The goal of Onionr is to provide a method of distributing information in a manner in which the difficulty of discovering the identity of those sending and receiving the information is greatly increased. In this section we detail what information we want to protect and who we're protecting it from.
 
 In this threat model, "protected" means available in plaintext only to those which it was intended, and regardless non-malleable
 
@@ -131,7 +134,7 @@ Onionr does not protect the following:
 
 ## Assumptions
 
-We assume that Tor onion services (v3) and I2P services cannot be trivially deanonymized, and that the underlying cryptographic primitives we employ cannot be broken in any manner faster than brute force unless a quantum computer is used. 
+We assume that Tor onion services (v3) and I2P services cannot be trivially deanonymized, and that the underlying cryptographic primitives we employ cannot be broken in any manner faster than brute force unless a quantum computer is used.
 
 Once quantum safe algorithms are more mature and have decent high level libraries, they will be deployed.
 

git-update.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+echo "This will git pull origin master, then update dependencies and default plugins."
+echo "Enter to continue, ctrl-c to stop."
+read
+
+git pull origin master
+pip3 install -r --require-hashes requirements.txt
+./onionr.sh reset-plugins

install/CentOS-install

@@ -0,0 +1,35 @@
+yum install epel-release
+yum clean all
+yum update -y
+
+yum group install "Development Tools"
+yum install centos-release-scl
+yum install wget zlib zlib-devel openssl-devel bzip2-devel libffi-devel python-gevent -y
+yum install rh-python38 rh-python38-python-devel
+scl enable rh-python38 bash
+yum update -y
+
+pip3 install --require-hashes -r requirements.txt
+chown nobody. ../
+su -l nobody -s /bin/bash
+scl enable rh-python38 bash
+./onionr.sh start
+
+Like a black hole, NSA pulls in every signal that comes near, but no electron is ever allowed to escape│
+│ -James Bamford                                                                                        │
+└───────────────────────────────────────────────────────────────────────────────────────────────────────┘
+[+] Tor is starting...
+[+] Jan 11 18:35:55.000 [notice] Bootstrapped 0%: Starting
+[+] Jan 11 18:35:56.000 [notice] Bootstrapped 10%: Finishing handshake with directory server
+[+] Jan 11 18:35:56.000 [notice] Bootstrapped 80%: Connecting to the Tor network
+[+] Jan 11 18:35:56.000 [notice] Bootstrapped 90%: Establishing a Tor circuit
+[+] Finished starting Tor.
+[+] Onionr v8.0.1 (x86_64) (API v2)
+[+] Private P2P Communication - GPLv3 - https://Onionr.net
+[+] CPython 3.8.6 on Linux-3.10.0-1062.4.1.el7.x86_64-x86_64-with-glibc2.2.5 3.10.0-1062.4.1.el7.x86_64
+[+] Onionr data dir: /root/.local/share/onionr/
+[+] Onionr daemon is running under 5186
+[+] Not binding to LAN due to no private network configured.
+[+] First run detected. Run openhome to get setup.
+
+./onionr.sh openhome

install/PKGBUILD

@@ -0,0 +1,60 @@
+# Maintainer: Aaron Esau <aur@aaronesau.com>
+pkgname="onionr-git"
+pkgver="0.1"
+pkgrel="1"
+conflicts=("onionr2")
+license=("GPL")
+arch=("i686" "x86_64")
+md5sums=("SKIP")
+url="https://onionr.net/"
+pkgdesc="anonymous P2P communication platform"
+source=("${pkgname}-${pkgver}::git+https://gitlab.com/beardog/onionr.git#branch=master")
+makedepends=("python" "python-setuptools" "python-pip")
+depends=("git" "curl" "tor")
+
+rinstall() {
+    if [ -f "$1" ]; then
+        install -D "$1" "$2/" "$3" "$4"
+        return 0
+    fi
+
+    for file in $(find "$1" -type f -printf '%P\n'); do
+        install -D "$1/$file" "$2/$file" "$3" "$4"
+    done
+
+    return 0
+}
+
+prepare() {
+    # pre_build
+
+    cd "${srcdir}/${pkgname}-${pkgver}"
+    
+}
+
+build() {
+    # build
+
+    cd "${srcdir}/${pkgname}-${pkgver}"
+    sh install/build.sh
+}
+
+check() {
+    # post_build
+
+    cd "${srcdir}/${pkgname}-${pkgver}"
+    
+}
+
+package() {
+    # "movements"
+    # ensure target directories exist
+    mkdir -p "$pkgdir//usr/bin/"
+    mkdir -p "$pkgdir//etc/systemd/system/"
+    mkdir -p "$pkgdir//usr/share/onionr"
+    
+    # copy files over and change perms
+    rinstall "${srcdir}/${pkgname}-${pkgver}/install/onionr" "${pkgdir}//usr/bin/" --mode=755 --owner="root" --group="root"
+    rinstall "${srcdir}/${pkgname}-${pkgver}/install/onionr.service" "${pkgdir}//etc/systemd/system/" --mode=644 --owner="root" --group="root"
+    rinstall "${srcdir}/${pkgname}-${pkgver}/." "${pkgdir}//usr/share/onionr" --mode=755 --owner="root" --group="root"
+}

install/README.md

@@ -0,0 +1,5 @@
+# Service/System Installation Notes
+
+Currently, the most reliable and well tested way to use Onionr is straight from the master branch of the repository, or from the released bundles.
+
+System services and system-wide installation is not supported well yet.

install/build.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+sudo pip3 install -r requirements.txt
+make plugins-reset
+find . -name '__pycache__' -type d | xargs rm -rf

install/create_release.py

@@ -0,0 +1,52 @@
+import os
+
+version = ''
+contents = ''
+
+with open('../onionr/onionr.py', 'r') as f:
+    contents = f.read()
+    version = contents.split("ONIONR_VERSION = '")[1].split("'")[0]
+
+print('Current Onionr release version is %s (MAJOR.MINOR.VERSION)\n' % version)
+
+new_version = input('Enter new version: ')
+
+try:
+    int(new_version.replace('.', ''))
+except:
+    print('Invalid version number, try again.')
+    exit(1337)
+
+confirm = input('Please confirm the version change from %s to %s (y/N): ' % (version, new_version))
+
+print('\n------\n')
+
+if confirm.lower().startswith('y'):
+    print('- Updating version in onionr.py')
+    
+    with open('../onionr/onionr.py', 'w+') as f:
+        f.write(contents.replace("ONIONR_VERSION = '%s'" % version, "ONIONR_VERSION = '%s'" % new_version))
+        
+    print('- Updating version in PKGBUILD')
+    
+    with open('../onionr/PKGBUILD', 'w+') as f:
+        f.write(f.read().replace("pkgver=%s" % version, "pkgver=%s" % new_version))
+    
+    print('- Committing changes')
+    
+    os.system('cd ..; git add onionr/onionr.py; git commit -m "Increment Onionr version to %s"' % new_version)
+    
+    print('- Adding tag')
+    
+    os.system('cd ..; git tag %s' % new_version)
+    
+    print('- Pushing changes')
+    
+    # os.system('cd ..; git push origin --tags')
+    
+    print('\n------\n\nAll done. Create a merge request into master at this link:\n\nhttps://gitlab.com/beardog/Onionr/merge_requests/new?merge_request%5Bsource_project_id%5D=5020889&merge_request%5Btarget_branch%5D=master&merge_request%5Btarget_project_id%5D=5020889')
+    
+    print('\nNOTE: The default configuration file was not changed. Please make sure it is not in dev mode, and that log.verbosity is "error".')
+else:
+    print('Change cancelled. No action has been taken.')
+

install/install_arch.sh

@@ -22,27 +22,31 @@ fi
 
 # install basic dependencies
 
-pacman --needed --noconfirm -S git curl python python-pip tor
+echo -e "\033[0;32mInstalling pacman dependencies...\033[0m"
+
+! ((pacman --needed --noconfirm -S git curl python python-pip tor > /dev/null) 2>&1 | grep -v warning 1>&2) | grep .
 
 # get the repository
 
+echo -e "\033[0;32mCloning Onionr repository...\033[0m"
+
 rm -rf "$OUTPUT_DIR" "$DATA_DIR" "$LOG_DIR"
 
-git clone https://gitlab.com/beardog/onionr "$OUTPUT_DIR"
+git clone --quiet https://gitlab.com/beardog/onionr "$OUTPUT_DIR" > /dev/null
 
 cd "$OUTPUT_DIR"
-git checkout "$BRANCH"
+git checkout -q "$BRANCH" > /dev/null
 
 # install python dependencies
 
-pip3 install --no-input -r "$OUTPUT_DIR/requirements.txt" --require-hashes
+echo -e "\033[0;32mInstalling pip dependencies...\033[0m"
 
-# create nologin onionr user if not exists
+pip3 install --no-input -r "$OUTPUT_DIR/requirements.txt" --require-hashes > /dev/null
 
-id -u onionr &>/dev/null || useradd -r -s /sbin/nologin onionr
+# set permissions on Onionr directory
 
 chmod 755 "$OUTPUT_DIR"
-chown -R onionr:onionr "$OUTPUT_DIR"
+chown -R root:root "$OUTPUT_DIR"
 
 # create directories
 
@@ -50,7 +54,7 @@ mkdir -p "$OUTPUT_DIR/onionr/data" "$LOG_DIR"
 mv "$OUTPUT_DIR/onionr/data" "$DATA_DIR"
 
 chmod -R 750 "$DATA_DIR" "$LOG_DIR"
-chown -R onionr:onionr "$DATA_DIR" "$LOG_DIR"
+chown -R root:root "$DATA_DIR" "$LOG_DIR"
 
 # create executable
 
@@ -61,6 +65,8 @@ chown root:root "$EXECUTABLE"
 
 # create systemd service
 
+echo -e "\033[0;32mCreating systemd unit...\033[0m"
+
 SERVICE='/etc/systemd/system/onionr.service'
 
 cp "$OUTPUT_DIR/install/onionr.service" "$SERVICE"
@@ -74,7 +80,7 @@ systemctl start onionr
 
 # pretty header thing
 
-"$EXECUTABLE" --header 'Onionr successfully installed.'
+"$EXECUTABLE" details
 
 # and we're good!
 

install/install_debian.sh

@@ -22,27 +22,31 @@ fi
 
 # install basic dependencies
 
-apt -y install git curl python3.7 python3-pip python3-setuptools tor
+echo -e "\033[0;32mInstalling apt dependencies...\033[0m"
+
+apt-get install -y git curl python3.7 python3-pip python3-setuptools tor > /dev/null
 
 # get the repository
 
+echo -e "\033[0;32mCloning Onionr repository...\033[0m"
+
 rm -rf "$OUTPUT_DIR" "$DATA_DIR" "$LOG_DIR"
 
-git clone https://gitlab.com/beardog/onionr "$OUTPUT_DIR"
+git clone --quiet https://gitlab.com/beardog/onionr "$OUTPUT_DIR" > /dev/null
 
 cd "$OUTPUT_DIR"
-git checkout "$BRANCH"
+git checkout -q "$BRANCH" > /dev/null
 
 # install python dependencies
 
-python3.7 -m pip install --no-input -r "$OUTPUT_DIR/requirements.txt" --require-hashes
+echo -e "\033[0;32mInstalling pip dependencies...\033[0m"
 
-# create nologin onionr user if not exists
+python3.7 -m pip install --no-input -r "$OUTPUT_DIR/requirements.txt" --require-hashes > /dev/null
 
-id -u onionr &>/dev/null || useradd -r -s /sbin/nologin onionr
+# set permissions on Onionr directory
 
 chmod 755 "$OUTPUT_DIR"
-chown -R onionr:onionr "$OUTPUT_DIR"
+chown -R root:root "$OUTPUT_DIR"
 
 # create directories
 
@@ -50,7 +54,7 @@ mkdir -p "$OUTPUT_DIR/onionr/data" "$LOG_DIR"
 mv "$OUTPUT_DIR/onionr/data" "$DATA_DIR"
 
 chmod -R 750 "$DATA_DIR" "$LOG_DIR"
-chown -R onionr:onionr "$DATA_DIR" "$LOG_DIR"
+chown -R root:root "$DATA_DIR" "$LOG_DIR"
 
 # create executable
 
@@ -61,6 +65,8 @@ chown root:root "$EXECUTABLE"
 
 # create systemd service
 
+echo -e "\033[0;32mCreating systemd unit...\033[0m"
+
 SERVICE='/etc/systemd/system/onionr.service'
 
 cp "$OUTPUT_DIR/install/onionr.service" "$SERVICE"

install/onionr

@@ -1,12 +1,16 @@
 #!/bin/sh
-
 set -e
 
-[ "root" != "$USER" ] && exec sudo $0 "$@"
+if [[ $EUID -eq 0 ]]; then
+    export ONIONR_HOME=${ONIONR_HOME:-/var/lib/onionr}
+    export LOG_DIR=${LOG_DIR:-/var/log/onionr}
+else
+    export ONIONR_HOME=${ONIONR_HOME:-${XDG_DATA_HOME:-$HOME/.local/share}/onionr}
+    export LOG_DIR=${LOG_DIR:-$ONIONR_HOME/logs}
+fi
 
-export OUTPUT_DIR=${OUTPUT_DIR:=/usr/share/onionr}
-export ONIONR_HOME=${ONIONR_HOME:=/etc/onionr}
-export LOG_DIR=${LOG_DIR:=/var/log/onionr}
+mkdir -p "$ONIONR_HOME" "$LOG_DIR"
 
-cd "$OUTPUT_DIR"
-exec su onionr -s /bin/sh -c "./onionr.sh ""$@"""
+chmod 0700 "$ONIONR_HOME" "$LOG_DIR"
+
+exec ${ONIONR_BASEDIR:-/usr/share/onionr}/onionr.sh "$@"

install/onionr.install

@@ -0,0 +1,48 @@
+
+pre_install {
+    # pre_install
+
+    cd "${srcdir}/${pkgname}-${pkgver}"
+    sh install/pre_install.sh
+
+}
+
+post_install {
+    # post_install
+
+    cd "${srcdir}/${pkgname}-${pkgver}"
+    sh install/post_install.sh
+
+}
+
+pre_upgrade {
+    # pre_upgrade
+
+    cd "${srcdir}/${pkgname}-${pkgver}"
+    
+
+}
+
+post_upgrade {
+    # post_upgrade
+
+    cd "${srcdir}/${pkgname}-${pkgver}"
+    
+
+}
+
+pre_remove {
+    # pre_remove
+
+    cd "${srcdir}/${pkgname}-${pkgver}"
+    
+
+}
+
+post_remove {
+    # post_remove
+
+    cd "${srcdir}/${pkgname}-${pkgver}"
+    
+
+}

install/onionr.service

@@ -1,15 +1,20 @@
 [Unit]
 Description=Onionr Daemon
-Requires=network.target tor.service
-After=network.target tor.service
+Documentation=https://onionr.net/docs/
+After=network-online.target
+Requires=network-online.target
 
 [Service]
-Environment="DATA_DIR=/usr/share/onionr"
-Environment="LOG_DIR=/var/log/onionr/"
-ExecStart=/usr/bin/onionr --start
-ExecStop=/usr/bin/onionr --stop
-Type=simple
-Restart=always
+Environment="ONIONR_HOME=/var/lib/onionr"
+Environment="LOG_DIR=/var/log/onionr"
+
+ExecStart=/usr/bin/onionr start
+ExecStop=/usr/bin/onionr stop
+
+KillMode=mixed
+KillSignal=SIGQUIT
+TimeoutStopSec=30s
+Restart=on-abnormal
 
 [Install]
-WantedBy=tor.service
+WantedBy=multi-user.target

install/post_build.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+sh run_tests.sh

install/post_install.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+systemctl daemon-reload
+systemctl enable onionr
+systemctl start onionr

install/pre_install.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+pip3 install --no-input -r "$OUTPUT_DIR/requirements.txt" --require-hashes > /dev/null

make-release.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/bash
+rm -rf dist
+mkdir dist
+mkdir dist/onionr/
+cp -t dist/onionr/ -r docs static-data install src onionr.sh start-daemon.sh setprofile.sh requirements.txt requirements-notifications.txt
+cp *.md dist/onionr/
+PIP_USER=false
+export PIP_USER
+cd dist
+tar -czvf onionr-no-deps.tar.gz onionr
+cd ..
+pip3 install --require-hashes -r requirements.txt --target=dist/onionr/src/
+pip3 install --require-hashes -r requirements-notifications.txt --target=dist/onionr/src/
+cd dist
+tar -czvf onionr.tar.gz onionr

onionr.sh

@@ -1,4 +1,9 @@
 #!/bin/sh
+ORIG_ONIONR_RUN_DIR=`pwd`
+export ORIG_ONIONR_RUN_DIR
+export PYTHONDONTWRITEBYTECODE=1
+export PYTHONUNBUFFERED=1
+export PYTHONOPTIMIZE="true"
 cd "$(dirname "$0")"
-cd onionr/
-./onionr.py "$@"
+cd src
+./__init__.py "$@"

onionr/api.py

@@ -1,531 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    This file handles all incoming http requests to the client, using Flask
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import random, threading, hmac, base64, time, os, json, socket
-from gevent.pywsgi import WSGIServer, WSGIHandler
-from gevent import Timeout
-import flask
-from flask import request, Response, abort, send_from_directory
-import core
-import onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config, onionrblockapi
-import httpapi
-from httpapi import friendsapi, profilesapi, configapi, miscpublicapi
-from onionrservices import httpheaders
-import onionr
-from onionrutils import bytesconverter, stringvalidators, epoch, mnemonickeys
-
-config.reload()
-class FDSafeHandler(WSGIHandler):
-    '''Our WSGI handler. Doesn't do much non-default except timeouts'''
-    def handle(self):
-       timeout = Timeout(60, exception=Exception)
-       timeout.start()
-       try:
-           WSGIHandler.handle(self)
-       except Timeout as ex:
-           raise
-
-def setBindIP(filePath=''):
-    '''Set a random localhost IP to a specified file (intended for private or public API localhost IPs)'''
-    if config.get('general.random_bind_ip', True):
-        hostOctets = [str(127), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF))]
-        data = '.'.join(hostOctets)
-        # Try to bind IP. Some platforms like Mac block non normal 127.x.x.x
-        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        try:
-            s.bind((data, 0))
-        except OSError:
-            # if mac/non-bindable, show warning and default to 127.0.0.1
-            logger.warn('Your platform appears to not support random local host addresses 127.x.x.x. Falling back to 127.0.0.1.')
-            data = '127.0.0.1'
-        s.close()
-    else:
-        data = '127.0.0.1'
-    if filePath != '':
-        with open(filePath, 'w') as bindFile:
-            bindFile.write(data)
-    return data
-
-class PublicAPI:
-    '''
-        The new client api server, isolated from the public api
-    '''
-    def __init__(self, clientAPI):
-        assert isinstance(clientAPI, API)
-        app = flask.Flask('PublicAPI')
-        self.i2pEnabled = config.get('i2p.host', False)
-        self.hideBlocks = [] # Blocks to be denied sharing
-        self.host = setBindIP(clientAPI._core.publicApiHostFile)
-        self.torAdder = clientAPI._core.hsAddress
-        self.i2pAdder = clientAPI._core.i2pAddress
-        self.bindPort = config.get('client.public.port')
-        self.lastRequest = 0
-        self.hitCount = 0 # total rec requests to public api since server started
-        logger.info('Running public api on %s:%s' % (self.host, self.bindPort))
-
-        @app.before_request
-        def validateRequest():
-            '''Validate request has the correct hostname'''
-            # If high security level, deny requests to public (HS should be disabled anyway for Tor, but might not be for I2P)
-            if config.get('general.security_level', default=1) > 0:
-                abort(403)
-            if type(self.torAdder) is None and type(self.i2pAdder) is None:
-                # abort if our hs addresses are not known
-                abort(403)
-            if request.host not in (self.i2pAdder, self.torAdder):
-                # Disallow connection if wrong HTTP hostname, in order to prevent DNS rebinding attacks
-                abort(403)
-            self.hitCount += 1 # raise hit count for valid requests
-
-        @app.after_request
-        def sendHeaders(resp):
-            '''Send api, access control headers'''
-            resp = httpheaders.set_default_onionr_http_headers(resp)
-            # Network API version
-            resp.headers['X-API'] = onionr.API_VERSION
-            self.lastRequest = epoch.get_rounded_epoch(roundS=5)
-            return resp
-
-        @app.route('/')
-        def banner():
-            # Display a bit of information to people who visit a node address in their browser
-            try:
-                with open('static-data/index.html', 'r') as html:
-                    resp = Response(html.read(), mimetype='text/html')
-            except FileNotFoundError:
-                resp = Response("")
-            return resp
-
-        @app.route('/getblocklist')
-        def getBlockList():
-            return httpapi.miscpublicapi.public_block_list(clientAPI, self, request)
-
-        @app.route('/getdata/<name>')
-        def getBlockData(name):
-            # Share data for a block if we have it
-            return httpapi.miscpublicapi.public_get_block_data(clientAPI, self, name)
-
-        @app.route('/www/<path:path>')
-        def wwwPublic(path):
-            # A way to share files directly over your .onion
-            if not config.get("www.public.run", True):
-                abort(403)
-            return send_from_directory(config.get('www.public.path', 'static-data/www/public/'), path)
-
-        @app.route('/ping')
-        def ping():
-            # Endpoint to test if nodes are up
-            return Response("pong!")
-
-        @app.route('/pex')
-        def peerExchange():
-            response = ','.join(clientAPI._core.listAdders(recent=3600))
-            if len(response) == 0:
-                response = ''
-            return Response(response)
-        
-        @app.route('/announce', methods=['post'])
-        def acceptAnnounce():
-            resp = httpapi.miscpublicapi.announce(clientAPI, request)
-            return resp
-
-        @app.route('/upload', methods=['post'])
-        def upload():
-            '''Accept file uploads. In the future this will be done more often than on creation 
-            to speed up block sync
-            '''
-            return httpapi.miscpublicapi.upload(clientAPI, request)
-
-        # Set instances, then startup our public api server
-        clientAPI.setPublicAPIInstance(self)
-        while self.torAdder == '':
-            clientAPI._core.refreshFirstStartVars()
-            self.torAdder = clientAPI._core.hsAddress
-            time.sleep(0.1)
-        self.httpServer = WSGIServer((self.host, self.bindPort), app, log=None, handler_class=FDSafeHandler)
-        self.httpServer.serve_forever()
-
-class API:
-    '''
-        Client HTTP api
-    '''
-
-    callbacks = {'public' : {}, 'private' : {}}
-
-    def __init__(self, onionrInst, debug, API_VERSION):
-        '''
-            Initialize the api server, preping variables for later use
-
-            This initialization defines all of the API entry points and handlers for the endpoints and errors
-            This also saves the used host (random localhost IP address) to the data folder in host.txt
-        '''
-
-        self.debug = debug
-        self._core = onionrInst.onionrCore
-        self.startTime = epoch.get_epoch()
-        self._crypto = onionrcrypto.OnionrCrypto(self._core)
-        app = flask.Flask(__name__)
-        bindPort = int(config.get('client.client.port', 59496))
-        self.bindPort = bindPort
-
-        # Be extremely mindful of this. These are endpoints available without a password
-        self.whitelistEndpoints = ('site', 'www', 'onionrhome', 'homedata', 'board', 'profiles', 'profilesindex', 
-        'boardContent', 'sharedContent', 'mail', 'mailindex', 'friends', 'friendsindex',
-        'clandestine', 'clandestineIndex')
-
-        self.clientToken = config.get('client.webpassword')
-        self.timeBypassToken = base64.b16encode(os.urandom(32)).decode()
-
-        self.publicAPI = None # gets set when the thread calls our setter... bad hack but kinda necessary with flask
-        #threading.Thread(target=PublicAPI, args=(self,)).start()
-        self.host = setBindIP(self._core.privateApiHostFile)
-        logger.info('Running api on %s:%s' % (self.host, self.bindPort))
-        self.httpServer = ''
-
-        self.queueResponse = {}
-        onionrInst.setClientAPIInst(self)
-        app.register_blueprint(friendsapi.friends)
-        app.register_blueprint(profilesapi.profile_BP)
-        app.register_blueprint(configapi.config_BP)
-        httpapi.load_plugin_blueprints(app)
-
-        @app.before_request
-        def validateRequest():
-            '''Validate request has set password and is the correct hostname'''
-            # For the purpose of preventing DNS rebinding attacks
-            if request.host != '%s:%s' % (self.host, self.bindPort):
-                abort(403)
-            if request.endpoint in self.whitelistEndpoints:
-                return
-            try:
-                if not hmac.compare_digest(request.headers['token'], self.clientToken):
-                    if not hmac.compare_digest(request.form['token'], self.clientToken):
-                        abort(403)
-            except KeyError:
-                if not hmac.compare_digest(request.form['token'], self.clientToken):
-                    abort(403)
-
-        @app.after_request
-        def afterReq(resp):
-            # Security headers
-            resp = httpheaders.set_default_onionr_http_headers(resp)
-            if request.endpoint == 'site':
-                resp.headers['Content-Security-Policy'] = "default-src 'none'; style-src data: 'unsafe-inline'; img-src data:"
-            else:
-                resp.headers['Content-Security-Policy'] = "default-src 'none'; script-src 'self'; object-src 'none'; style-src 'self'; img-src 'self'; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'self'"
-            return resp
-
-        @app.route('/board/', endpoint='board')
-        def loadBoard():
-            return send_from_directory('static-data/www/board/', "index.html")
-
-        @app.route('/mail/<path:path>', endpoint='mail')
-        def loadMail(path):
-            return send_from_directory('static-data/www/mail/', path)
-        @app.route('/mail/', endpoint='mailindex')
-        def loadMailIndex():
-            return send_from_directory('static-data/www/mail/', 'index.html')
-
-        @app.route('/clandestine/<path:path>', endpoint='clandestine')
-        def loadClandestine(path):
-            return send_from_directory('static-data/www/clandestine/', path)
-        @app.route('/clandestine/', endpoint='clandestineIndex')
-        def loadClandestineIndex():
-            return send_from_directory('static-data/www/clandestine/', 'index.html')
-        
-        @app.route('/friends/<path:path>', endpoint='friends')
-        def loadContacts(path):
-            return send_from_directory('static-data/www/friends/', path)
-
-        @app.route('/friends/', endpoint='friendsindex')
-        def loadContacts():
-            return send_from_directory('static-data/www/friends/', 'index.html')
-
-        @app.route('/profiles/<path:path>', endpoint='profiles')
-        def loadContacts(path):
-            return send_from_directory('static-data/www/profiles/', path)
-
-        @app.route('/profiles/', endpoint='profilesindex')
-        def loadContacts():
-            return send_from_directory('static-data/www/profiles/', 'index.html')
-        
-        @app.route('/serviceactive/<pubkey>')
-        def serviceActive(pubkey):
-            try:
-                if pubkey in self._core.onionrInst.communicatorInst.active_services:
-                    return Response('true')
-            except AttributeError as e:
-                pass
-            return Response('false')
-
-        @app.route('/board/<path:path>', endpoint='boardContent')
-        def boardContent(path):
-            return send_from_directory('static-data/www/board/', path)
-        @app.route('/shared/<path:path>', endpoint='sharedContent')
-        def sharedContent(path):
-            return send_from_directory('static-data/www/shared/', path)
-
-        @app.route('/', endpoint='onionrhome')
-        def hello():
-            # ui home
-            return send_from_directory('static-data/www/private/', 'index.html')
-        
-        @app.route('/private/<path:path>', endpoint='homedata')
-        def homedata(path):
-            return send_from_directory('static-data/www/private/', path)
-
-        @app.route('/www/<path:path>', endpoint='www')
-        def wwwPublic(path):
-            if not config.get("www.private.run", True):
-                abort(403)
-            return send_from_directory(config.get('www.private.path', 'static-data/www/private/'), path)
-
-        @app.route('/hitcount')
-        def get_hit_count():
-            return Response(str(self.publicAPI.hitCount))
-
-        @app.route('/queueResponseAdd/<name>', methods=['post'])
-        def queueResponseAdd(name):
-            # Responses from the daemon. TODO: change to direct var access instead of http endpoint
-            self.queueResponse[name] = request.form['data']
-            return Response('success')
-        
-        @app.route('/queueResponse/<name>')
-        def queueResponse(name):
-            # Fetch a daemon queue response
-            resp = 'failure'
-            try:
-                resp = self.queueResponse[name]
-            except KeyError:
-                pass
-            else:
-                del self.queueResponse[name]
-            if resp == 'failure':
-                return resp, 404
-            else:
-                return resp
-            
-        @app.route('/ping')
-        def ping():
-            # Used to check if client api is working
-            return Response("pong!")
-        
-        @app.route('/getblocksbytype/<name>')
-        def getBlocksByType(name):
-            blocks = self._core.getBlocksByType(name)
-            return Response(','.join(blocks))
-        
-        @app.route('/getblockbody/<name>')
-        def getBlockBodyData(name):
-            resp = ''
-            if stringvalidators.validate_hash(name):
-                try:
-                    resp = onionrblockapi.Block(name, decrypt=True).bcontent
-                except TypeError:
-                    pass
-            else:
-                abort(404)
-            return Response(resp)
-        
-        @app.route('/getblockdata/<name>')
-        def getData(name):
-            resp = ""
-            if stringvalidators.validate_hash(name):
-                if name in self._core.getBlockList():
-                    try:
-                        resp = self.getBlockData(name, decrypt=True)
-                    except ValueError:
-                        pass
-                else:
-                    abort(404)
-            else:
-                abort(404)
-            return Response(resp)
-
-        @app.route('/getblockheader/<name>')
-        def getBlockHeader(name):
-            resp = self.getBlockData(name, decrypt=True, headerOnly=True)
-            return Response(resp)
-
-        @app.route('/lastconnect')
-        def lastConnect():
-            return Response(str(self.publicAPI.lastRequest))
-
-        @app.route('/site/<name>', endpoint='site')
-        def site(name):
-            bHash = name
-            resp = 'Not Found'
-            if stringvalidators.validate_hash(bHash):
-                try:
-                    resp = onionrblockapi.Block(bHash).bcontent
-                except onionrexceptions.NoDataAvailable:
-                    abort(404)
-                except TypeError:
-                    pass
-                try:
-                    resp = base64.b64decode(resp)
-                except:
-                    pass
-            if resp == 'Not Found' or not resp:
-                abort(404)
-            return Response(resp)
-
-        @app.route('/waitforshare/<name>', methods=['post'])
-        def waitforshare(name):
-            '''Used to prevent the **public** api from sharing blocks we just created'''
-            assert name.isalnum()
-            if name in self.publicAPI.hideBlocks:
-                self.publicAPI.hideBlocks.remove(name)
-                return Response("removed")
-            else:
-                self.publicAPI.hideBlocks.append(name)
-                return Response("added")
-
-        @app.route('/shutdown')
-        def shutdown():
-            try:
-                self.publicAPI.httpServer.stop()
-                self.httpServer.stop()
-            except AttributeError:
-                pass
-            return Response("bye")
-
-        @app.route('/shutdownclean')
-        def shutdownClean():
-            # good for calling from other clients
-            self._core.daemonQueueAdd('shutdown')
-            return Response("bye")
-        
-        @app.route('/getstats')
-        def getStats():
-            # returns node stats
-            #return Response("disabled")
-            while True:
-                try:    
-                    return Response(self._core.serializer.getStats())
-                except AttributeError:
-                    pass
-        
-        @app.route('/getuptime')
-        def showUptime():
-            return Response(str(self.getUptime()))
-        
-        @app.route('/getActivePubkey')
-        def getActivePubkey():
-            return Response(self._core._crypto.pubKey)
-
-        @app.route('/getHumanReadable/<name>')
-        def getHumanReadable(name):
-            return Response(mnemonickeys.get_human_readable_ID(name))
-
-        @app.route('/insertblock', methods=['POST'])
-        def insertBlock():
-            encrypt = False
-            bData = request.get_json(force=True)
-            message = bData['message']
-
-            # Detect if message (block body) is not specified
-            if type(message) is None:
-                return 'failure', 406
-
-            subject = 'temp'
-            encryptType = ''
-            sign = True
-            meta = {}
-            to = ''
-            try:
-                if bData['encrypt']:
-                    to = bData['to']
-                    encrypt = True
-                    encryptType = 'asym'
-            except KeyError:
-                pass
-            try:
-                if not bData['sign']:
-                    sign = False
-            except KeyError:
-                pass
-            try:
-                bType = bData['type']
-            except KeyError:
-                bType = 'bin'
-            try:
-                meta = json.loads(bData['meta'])
-            except KeyError:
-                pass
-            threading.Thread(target=self._core.insertBlock, args=(message,), kwargs={'header': bType, 'encryptType': encryptType, 'sign':sign, 'asymPeer': to, 'meta': meta}).start()
-            return Response('success')
-
-        self.httpServer = WSGIServer((self.host, bindPort), app, log=None, handler_class=FDSafeHandler)
-        self.httpServer.serve_forever()
-
-    def setPublicAPIInstance(self, inst):
-        assert isinstance(inst, PublicAPI)
-        self.publicAPI = inst
-
-    def validateToken(self, token):
-        '''
-            Validate that the client token matches the given token. Used to prevent CSRF and data exfiltration
-        '''
-        if len(self.clientToken) == 0:
-            logger.error("client password needs to be set")
-            return False
-        try:
-            if not hmac.compare_digest(self.clientToken, token):
-                return False
-            else:
-                return True
-        except TypeError:
-            return False
-
-    def getUptime(self):
-        while True:
-            try:
-                return epoch.get_epoch() - self.startTime
-            except (AttributeError, NameError):
-                # Don't error on race condition with startup
-                pass
-    
-    def getBlockData(self, bHash, decrypt=False, raw=False, headerOnly=False):
-        assert stringvalidators.validate_hash(bHash)
-        bl = onionrblockapi.Block(bHash, core=self._core)
-        if decrypt:
-            bl.decrypt()
-            if bl.isEncrypted and not bl.decrypted:
-                raise ValueError
-
-        if not raw:
-            if not headerOnly:
-                retData = {'meta':bl.bheader, 'metadata': bl.bmetadata, 'content': bl.bcontent}
-                for x in list(retData.keys()):
-                    try:
-                        retData[x] = retData[x].decode()
-                    except AttributeError:
-                        pass
-            else:
-                validSig = False
-                signer = bytesconverter.bytes_to_str(bl.signer)
-                if bl.isSigned() and stringvalidators.validate_pub_key(signer) and bl.isSigner(signer):
-                    validSig = True                    
-                bl.bheader['validSig'] = validSig
-                bl.bheader['meta'] = ''
-                retData = {'meta': bl.bheader, 'metadata': bl.bmetadata}
-            return json.dumps(retData)
-        else:
-            return bl.raw

onionr/blockimporter.py

@@ -1,51 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    Import block data and save it
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import core, onionrexceptions, logger
-from onionrutils import validatemetadata, blockmetadata
-def importBlockFromData(content, coreInst):
-    retData = False
-
-    dataHash = coreInst._crypto.sha3Hash(content)
-
-    if coreInst._blacklist.inBlacklist(dataHash):
-        raise onionrexceptions.BlacklistedBlock('%s is a blacklisted block' % (dataHash,))
-
-    if not isinstance(coreInst, core.Core):
-        raise Exception("coreInst must be an Onionr core instance")
-
-    try:
-        content = content.encode()
-    except AttributeError:
-        pass
-
-    metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
-    metadata = metas[0]
-    if validatemetadata.validate_metadata(coreInst, metadata, metas[2]): # check if metadata is valid
-        if coreInst._crypto.verifyPow(content): # check if POW is enough/correct
-            logger.info('Block passed proof, saving.', terminal=True)
-            try:
-                blockHash = coreInst.setData(content)
-            except onionrexceptions.DiskAllocationReached:
-                pass
-            else:
-                coreInst.addToBlockDB(blockHash, dataSaved=True)
-                blockmetadata.process_block_metadata(coreInst, blockHash) # caches block metadata values to block database
-                retData = True
-    return retData

onionr/communicator.py

@@ -1,393 +0,0 @@
-#!/usr/bin/env python3
-'''
-    Onionr - Private P2P Communication
-
-    This file contains both the OnionrCommunicate class for communcating with peers
-    and code to operate as a daemon, getting commands from the command queue database (see core.Core.daemonQueue)
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import sys, os, time
-import core, config, logger, onionr
-import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block
-from communicatorutils import servicecreator, onionrcommunicatortimers
-from communicatorutils import downloadblocks, lookupblocks, lookupadders
-from communicatorutils import servicecreator, connectnewpeers, uploadblocks
-from communicatorutils import daemonqueuehandler, announcenode, deniableinserts
-from communicatorutils import cooldownpeer, housekeeping, netcheck
-from onionrutils import localcommand, epoch, basicrequests
-from etc import humanreadabletime
-import onionrservices, onionr, onionrproofs
-
-OnionrCommunicatorTimers = onionrcommunicatortimers.OnionrCommunicatorTimers
-
-config.reload()
-class OnionrCommunicatorDaemon:
-    def __init__(self, onionrInst, proxyPort, developmentMode=config.get('general.dev_mode', False)):
-        onionrInst.communicatorInst = self
-        # configure logger and stuff
-        onionr.Onionr.setupConfig('data/', self = self)
-        self.proxyPort = proxyPort
-
-        self.isOnline = True # Assume we're connected to the internet
-
-        # list of timer instances
-        self.timers = []
-
-        # initialize core with Tor socks port being 3rd argument
-        self.proxyPort = proxyPort
-        self._core = onionrInst.onionrCore
-
-        self.blocksToUpload = []
-
-        # loop time.sleep delay in seconds
-        self.delay = 1
-
-        # lists of connected peers and peers we know we can't reach currently
-        self.onlinePeers = []
-        self.offlinePeers = []
-        self.cooldownPeer = {}
-        self.connectTimes = {}
-        self.peerProfiles = [] # list of peer's profiles (onionrpeers.PeerProfile instances)
-        self.newPeers = [] # Peers merged to us. Don't add to db until we know they're reachable
-        self.announceProgress = {}
-        self.announceCache = {}
-
-        # amount of threads running by name, used to prevent too many
-        self.threadCounts = {}
-
-        # set true when shutdown command received
-        self.shutdown = False
-
-        # list of new blocks to download, added to when new block lists are fetched from peers
-        self.blockQueue = {}
-
-        # list of blocks currently downloading, avoid s
-        self.currentDownloading = []
-
-        # timestamp when the last online node was seen
-        self.lastNodeSeen = None
-
-        # Dict of time stamps for peer's block list lookup times, to avoid downloading full lists all the time
-        self.dbTimestamps = {}
-
-        # Clear the daemon queue for any dead messages
-        if os.path.exists(self._core.queueDB):
-            self._core.clearDaemonQueue()
-
-        # Loads in and starts the enabled plugins
-        plugins.reload()
-
-        # time app started running for info/statistics purposes
-        self.startTime = epoch.get_epoch()
-
-        if developmentMode:
-            OnionrCommunicatorTimers(self, self.heartbeat, 30)
-
-        # Set timers, function reference, seconds
-        # requiresPeer True means the timer function won't fire if we have no connected peers
-        peerPoolTimer = OnionrCommunicatorTimers(self, self.getOnlinePeers, 60, maxThreads=1)
-        OnionrCommunicatorTimers(self, self.runCheck, 2, maxThreads=1)
-
-        # Timers to periodically lookup new blocks and download them
-        OnionrCommunicatorTimers(self, self.lookupBlocks, self._core.config.get('timers.lookupBlocks', 25), requiresPeer=True, maxThreads=1)
-        OnionrCommunicatorTimers(self, self.getBlocks, self._core.config.get('timers.getBlocks', 30), requiresPeer=True, maxThreads=2)
-
-        # Timer to reset the longest offline peer so contact can be attempted again
-        OnionrCommunicatorTimers(self, self.clearOfflinePeer, 58)
-
-        # Timer to cleanup old blocks
-        blockCleanupTimer = OnionrCommunicatorTimers(self, housekeeping.clean_old_blocks, 65, myArgs=[self])
-
-        # Timer to discover new peers
-        OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True)
-
-        # Timer for adjusting which peers we actively communicate to at any given time, to avoid over-using peers
-        OnionrCommunicatorTimers(self, cooldownpeer.cooldown_peer, 30, myArgs=[self], requiresPeer=True)
-
-        # Timer to read the upload queue and upload the entries to peers
-        OnionrCommunicatorTimers(self, self.uploadBlock, 5, requiresPeer=True, maxThreads=1)
-
-        # Timer to process the daemon command queue
-        OnionrCommunicatorTimers(self, self.daemonCommands, 6, maxThreads=3)
-
-        # Timer that kills Onionr if the API server crashes
-        #OnionrCommunicatorTimers(self, self.detectAPICrash, 30, maxThreads=1)
-
-        # Setup direct connections
-        if config.get('general.socket_servers', False):
-            self.services = onionrservices.OnionrServices(self._core)
-            self.active_services = []
-            self.service_greenlets = []
-            OnionrCommunicatorTimers(self, servicecreator.service_creator, 5, maxThreads=50, myArgs=[self])
-        else:
-            self.services = None
-        
-        # This timer creates deniable blocks, in an attempt to further obfuscate block insertion metadata
-        if config.get('general.insert_deniable_blocks', True):
-            deniableBlockTimer = OnionrCommunicatorTimers(self, deniableinserts.insert_deniable_block, 180, myArgs=[self], requiresPeer=True, maxThreads=1)
-            deniableBlockTimer.count = (deniableBlockTimer.frequency - 175)
-
-        # Timer to check for connectivity, through Tor to various high-profile onion services
-        netCheckTimer = OnionrCommunicatorTimers(self, netcheck.net_check, 600, myArgs=[self])
-
-        # Announce the public API server transport address to other nodes if security level allows
-        if config.get('general.security_level', 1) == 0 and config.get('general.announce_node', True):
-            # Default to high security level incase config breaks
-            announceTimer = OnionrCommunicatorTimers(self, announcenode.announce_node, 3600, myArgs=[self], requiresPeer=True, maxThreads=1)
-            announceTimer.count = (announceTimer.frequency - 120)
-        else:
-            logger.debug('Will not announce node.')
-        
-        # Timer to delete malfunctioning or long-dead peers
-        cleanupTimer = OnionrCommunicatorTimers(self, self.peerCleanup, 300, requiresPeer=True)
-
-        # Timer to cleanup dead ephemeral forward secrecy keys 
-        forwardSecrecyTimer = OnionrCommunicatorTimers(self, housekeeping.clean_keys, 15, myArgs=[self], maxThreads=1)
-
-        # Adjust initial timer triggers
-        peerPoolTimer.count = (peerPoolTimer.frequency - 1)
-        cleanupTimer.count = (cleanupTimer.frequency - 60)
-        blockCleanupTimer.count = (blockCleanupTimer.frequency - 5)
-
-        # Main daemon loop, mainly for calling timers, don't do any complex operations here to avoid locking
-        try:
-            while not self.shutdown:
-                for i in self.timers:
-                    if self.shutdown:
-                        break
-                    i.processTimer()
-                time.sleep(self.delay)
-                # Debug to print out used FDs (regular and net)
-                #proc = psutil.Process()
-                #print(proc.open_files(), len(psutil.net_connections()))
-        except KeyboardInterrupt:
-            self.shutdown = True
-            pass
-
-        logger.info('Goodbye. (Onionr is cleaning up, and will exit)', terminal=True)
-        try:
-            self.service_greenlets
-        except AttributeError:
-            pass
-        else:
-            for server in self.service_greenlets:
-                server.stop()
-        localcommand.local_command(self._core, 'shutdown') # shutdown the api
-        time.sleep(0.5)
-
-    def lookupAdders(self):
-        '''Lookup new peer addresses'''
-        lookupadders.lookup_new_peer_transports_with_communicator(self)
-
-    def lookupBlocks(self):
-        '''Lookup new blocks & add them to download queue'''
-        lookupblocks.lookup_blocks_from_communicator(self)
-
-    def getBlocks(self):
-        '''download new blocks in queue'''
-        downloadblocks.download_blocks_from_communicator(self)
-
-    def decrementThreadCount(self, threadName):
-        '''Decrement amount of a thread name if more than zero, called when a function meant to be run in a thread ends'''
-        try:
-            if self.threadCounts[threadName] > 0:
-                self.threadCounts[threadName] -= 1
-        except KeyError:
-            pass
-
-    def pickOnlinePeer(self):
-        '''randomly picks peer from pool without bias (using secrets module)'''
-        retData = ''
-        while True:
-            peerLength = len(self.onlinePeers)
-            if peerLength <= 0:
-                break
-            try:
-                # get a random online peer, securely. May get stuck in loop if network is lost or if all peers in pool magically disconnect at once
-                retData = self.onlinePeers[self._core._crypto.secrets.randbelow(peerLength)]
-            except IndexError:
-                pass
-            else:
-                break
-        return retData
-
-    def clearOfflinePeer(self):
-        '''Removes the longest offline peer to retry later'''
-        try:
-            removed = self.offlinePeers.pop(0)
-        except IndexError:
-            pass
-        else:
-            logger.debug('Removed ' + removed + ' from offline list, will try them again.')
-        self.decrementThreadCount('clearOfflinePeer')
-
-    def getOnlinePeers(self):
-        '''
-            Manages the self.onlinePeers attribute list, connects to more peers if we have none connected
-        '''
-
-        logger.debug('Refreshing peer pool...')
-        maxPeers = int(config.get('peers.max_connect', 10))
-        needed = maxPeers - len(self.onlinePeers)
-
-        for i in range(needed):
-            if len(self.onlinePeers) == 0:
-                self.connectNewPeer(useBootstrap=True)
-            else:
-                self.connectNewPeer()
-
-            if self.shutdown:
-                break
-        else:
-            if len(self.onlinePeers) == 0:
-                logger.debug('Couldn\'t connect to any peers.' + (' Last node seen %s ago.' % humanreadabletime.human_readable_time(time.time() - self.lastNodeSeen) if not self.lastNodeSeen is None else ''), terminal=True)
-            else:
-                self.lastNodeSeen = time.time()
-        self.decrementThreadCount('getOnlinePeers')
-
-    def addBootstrapListToPeerList(self, peerList):
-        '''
-            Add the bootstrap list to the peer list (no duplicates)
-        '''
-        for i in self._core.bootstrapList:
-            if i not in peerList and i not in self.offlinePeers and i != self._core.hsAddress and len(str(i).strip()) > 0:
-                peerList.append(i)
-                self._core.addAddress(i)
-
-    def connectNewPeer(self, peer='', useBootstrap=False):
-        '''Adds a new random online peer to self.onlinePeers'''
-        connectnewpeers.connect_new_peer_to_communicator(self, peer, useBootstrap)
-
-    def removeOnlinePeer(self, peer):
-        '''Remove an online peer'''
-        try:
-            del self.connectTimes[peer]
-        except KeyError:
-            pass
-        try:
-            del self.dbTimestamps[peer]
-        except KeyError:
-            pass
-        try:
-            self.onlinePeers.remove(peer)
-        except ValueError:
-            pass
-
-    def peerCleanup(self):
-        '''This just calls onionrpeers.cleanupPeers, which removes dead or bad peers (offline too long, too slow)'''
-        onionrpeers.peerCleanup(self._core)
-        self.decrementThreadCount('peerCleanup')
-
-    def printOnlinePeers(self):
-        '''logs online peer list'''
-        if len(self.onlinePeers) == 0:
-            logger.warn('No online peers', terminal=True)
-        else:
-            logger.info('Online peers:', terminal=True)
-            for i in self.onlinePeers:
-                score = str(self.getPeerProfileInstance(i).score)
-                logger.info(i + ', score: ' + score, terminal=True)
-
-    def peerAction(self, peer, action, data='', returnHeaders=False):
-        '''Perform a get request to a peer'''
-        if len(peer) == 0:
-            return False
-        #logger.debug('Performing ' + action + ' with ' + peer + ' on port ' + str(self.proxyPort))
-        url = 'http://%s/%s' % (peer, action)
-        if len(data) > 0:
-            url += '&data=' + data
-
-        self._core.setAddressInfo(peer, 'lastConnectAttempt', epoch.get_epoch()) # mark the time we're trying to request this peer
-
-        retData = basicrequests.do_get_request(self._core, url, port=self.proxyPort)
-        # if request failed, (error), mark peer offline
-        if retData == False:
-            try:
-                self.getPeerProfileInstance(peer).addScore(-10)
-                self.removeOnlinePeer(peer)
-                if action != 'ping' and not self.shutdown:
-                    logger.warn('Lost connection to ' + peer, terminal=True)
-                    self.getOnlinePeers() # Will only add a new peer to pool if needed
-            except ValueError:
-                pass
-        else:
-            self._core.setAddressInfo(peer, 'lastConnect', epoch.get_epoch())
-            self.getPeerProfileInstance(peer).addScore(1)
-        return retData # If returnHeaders, returns tuple of data, headers. if not, just data string
-
-    def getPeerProfileInstance(self, peer):
-        '''Gets a peer profile instance from the list of profiles, by address name'''
-        for i in self.peerProfiles:
-            # if the peer's profile is already loaded, return that
-            if i.address == peer:
-                retData = i
-                break
-        else:
-            # if the peer's profile is not loaded, return a new one. connectNewPeer adds it the list on connect
-            retData = onionrpeers.PeerProfiles(peer, self._core)
-        return retData
-
-    def getUptime(self):
-        return epoch.get_epoch() - self.startTime
-
-    def heartbeat(self):
-        '''Show a heartbeat debug message'''
-        logger.debug('Heartbeat. Node running for %s.' % humanreadabletime.human_readable_time(self.getUptime()))
-        self.decrementThreadCount('heartbeat')
-
-    def daemonCommands(self):
-        '''
-            Process daemon commands from daemonQueue
-        '''
-        daemonqueuehandler.handle_daemon_commands(self)
-
-    def uploadBlock(self):
-        '''Upload our block to a few peers'''
-        uploadblocks.upload_blocks_from_communicator(self)
-
-    def announce(self, peer):
-        '''Announce to peers our address'''
-        if announcenode.announce_node(self) == False:
-            logger.warn('Could not introduce node.', terminal=True)
-
-    def detectAPICrash(self):
-        '''exit if the api server crashes/stops'''
-        if localcommand.local_command(self._core, 'ping', silent=False) not in ('pong', 'pong!'):
-            for i in range(300):
-                if localcommand.local_command(self._core, 'ping') in ('pong', 'pong!') or self.shutdown:
-                    break # break for loop
-                time.sleep(1)
-            else:
-                # This executes if the api is NOT detected to be running
-                events.event('daemon_crash', onionr = self._core.onionrInst, data = {})
-                logger.fatal('Daemon detected API crash (or otherwise unable to reach API after long time), stopping...', terminal=True)
-                self.shutdown = True
-        self.decrementThreadCount('detectAPICrash')
-
-    def runCheck(self):
-        if run_file_exists(self):
-            logger.debug('Status check; looks good.')
-
-        self.decrementThreadCount('runCheck')
-
-def startCommunicator(onionrInst, proxyPort):
-    OnionrCommunicatorDaemon(onionrInst, proxyPort)
-
-def run_file_exists(daemon):
-    if os.path.isfile(daemon._core.dataDir + '.runcheck'):
-        os.remove(daemon._core.dataDir + '.runcheck')
-        return True
-    return False

onionr/communicatorutils/README.md

@@ -1,33 +0,0 @@
-# communicatorutils
-
-The files in this submodule handle various subtasks and utilities for the onionr communicator.
-
-## Files:
-
-announcenode.py: Uses a communicator instance to announce our transport address to connected nodes
-
-connectnewpeers.py: takes a communicator instance and has it connect to as many peers as needed, and/or to a new specified peer.
-
-cooldownpeer.py: randomly selects a connected peer in a communicator and disconnects them for the purpose of security and network balancing.
-
-daemonqueuehandler.py: checks for new commands in the daemon queue and processes them accordingly.
-
-deniableinserts.py: insert fake blocks with the communicator for plausible deniability
-
-downloadblocks.py: iterates a communicator instance's block download queue and attempts to download the blocks from online peers
-
-housekeeping.py: cleans old blocks and forward secrecy keys
-
-lookupadders.py: ask connected peers to share their list of peer transport addresses
-
-lookupblocks.py: lookup new blocks from connected peers from the communicator
-
-netcheck.py: check if the node is online based on communicator status and onion server ping results
-
-onionrcommunicataortimers.py: create a timer for a function to be launched on an interval. Control how many possible instances of a timer may be running a function at once and control if the timer should be ran in a thread or not.
-
-proxypicker.py: returns a string name for the appropriate proxy to be used with a particular peer transport address.
-
-servicecreator.py: iterate connection blocks and create new direct connection servers for them.
-
-uploadblocks.py: iterate a communicator's upload queue and upload the blocks to connected peers

onionr/communicatorutils/announcenode.py

@@ -1,85 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    Use a communicator instance to announce our transport address to connected nodes
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import base64
-import onionrproofs, logger
-from etc import onionrvalues
-from onionrutils import basicrequests, bytesconverter
-
-def announce_node(daemon):
-    '''Announce our node to our peers'''
-    ov = onionrvalues.OnionrValues()
-    retData = False
-    announceFail = False
-    
-    # Do not let announceCache get too large
-    if len(daemon.announceCache) >= 10000:
-        daemon.announceCache.popitem()
-
-    if daemon._core.config.get('general.security_level', 0) == 0:
-        # Announce to random online peers
-        for i in daemon.onlinePeers:
-            if not i in daemon.announceCache and not i in daemon.announceProgress:
-                peer = i
-                break
-        else:
-            peer = daemon.pickOnlinePeer()
-
-        for x in range(1):
-            if x == 1 and daemon._core.config.get('i2p.host'):
-                ourID = daemon._core.config.get('i2p.own_addr').strip()
-            else:
-                ourID = daemon._core.hsAddress.strip()
-
-            url = 'http://' + peer + '/announce'
-            data = {'node': ourID}
-
-            combinedNodes = ourID + peer
-            if ourID != 1:
-                #TODO: Extend existingRand for i2p
-                existingRand = bytesconverter.bytes_to_str(daemon._core.getAddressInfo(peer, 'powValue'))
-                # Reset existingRand if it no longer meets the minimum POW
-                if type(existingRand) is type(None) or not existingRand.endswith('0' * ov.announce_pow):
-                    existingRand = ''
-
-            if peer in daemon.announceCache:
-                data['random'] = daemon.announceCache[peer]
-            elif len(existingRand) > 0:
-                data['random'] = existingRand
-            else:
-                daemon.announceProgress[peer] = True
-                proof = onionrproofs.DataPOW(combinedNodes, forceDifficulty=ov.announce_pow)
-                del daemon.announceProgress[peer]
-                try:
-                    data['random'] = base64.b64encode(proof.waitForResult()[1])
-                except TypeError:
-                    # Happens when we failed to produce a proof
-                    logger.error("Failed to produce a pow for announcing to " + peer)
-                    announceFail = True
-                else:
-                    daemon.announceCache[peer] = data['random']
-            if not announceFail:
-                logger.info('Announcing node to ' + url)
-                if basicrequests.do_post_request(daemon._core, url, data) == 'Success':
-                    logger.info('Successfully introduced node to ' + peer, terminal=True)
-                    retData = True
-                    daemon._core.setAddressInfo(peer, 'introduced', 1)
-                    daemon._core.setAddressInfo(peer, 'powValue', data['random'])
-    daemon.decrementThreadCount('announce_node')
-    return retData

onionr/communicatorutils/connectnewpeers.py

@@ -1,92 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    Connect a new peer to our communicator instance. Does so randomly if no peer is specified
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import time, sys
-import onionrexceptions, logger, onionrpeers
-from utils import networkmerger
-from onionrutils import stringvalidators, epoch
-# secrets module was added into standard lib in 3.6+
-if sys.version_info[0] == 3 and sys.version_info[1] < 6:
-    from dependencies import secrets
-elif sys.version_info[0] == 3 and sys.version_info[1] >= 6:
-    import secrets
-def connect_new_peer_to_communicator(comm_inst, peer='', useBootstrap=False):
-    config = comm_inst._core.config
-    retData = False
-    tried = comm_inst.offlinePeers
-    if peer != '':
-        if stringvalidators.validate_transport(peer):
-            peerList = [peer]
-        else:
-            raise onionrexceptions.InvalidAddress('Will not attempt connection test to invalid address')
-    else:
-        peerList = comm_inst._core.listAdders()
-
-    mainPeerList = comm_inst._core.listAdders()
-    peerList = onionrpeers.getScoreSortedPeerList(comm_inst._core)
-
-    # If we don't have enough peers connected or random chance, select new peers to try
-    if len(peerList) < 8 or secrets.randbelow(4) == 3:
-        tryingNew = []
-        for x in comm_inst.newPeers:
-            if x not in peerList:
-                peerList.append(x)
-                tryingNew.append(x)
-        for i in tryingNew:
-            comm_inst.newPeers.remove(i)
-    
-    if len(peerList) == 0 or useBootstrap:
-        # Avoid duplicating bootstrap addresses in peerList
-        comm_inst.addBootstrapListToPeerList(peerList)
-
-    for address in peerList:
-        if not config.get('tor.v3onions') and len(address) == 62:
-            continue
-        # Don't connect to our own address
-        if address == comm_inst._core.hsAddress:
-            continue
-        # Don't connect to invalid address or if its already been tried/connected, or if its cooled down
-        if len(address) == 0 or address in tried or address in comm_inst.onlinePeers or address in comm_inst.cooldownPeer:
-            continue
-        if comm_inst.shutdown:
-            return
-        # Ping a peer,
-        if comm_inst.peerAction(address, 'ping') == 'pong!':
-            time.sleep(0.1)
-            if address not in mainPeerList:
-                # Add a peer to our list if it isn't already since it successfully connected
-                networkmerger.mergeAdders(address, comm_inst._core)
-            if address not in comm_inst.onlinePeers:
-                logger.info('Connected to ' + address, terminal=True)
-                comm_inst.onlinePeers.append(address)
-                comm_inst.connectTimes[address] = epoch.get_epoch()
-            retData = address
-
-            # add peer to profile list if they're not in it
-            for profile in comm_inst.peerProfiles:
-                if profile.address == address:
-                    break
-            else:
-                comm_inst.peerProfiles.append(onionrpeers.PeerProfiles(address, comm_inst._core))
-            break
-        else:
-            # Mark a peer as tried if they failed to respond to ping
-            tried.append(address)
-            logger.debug('Failed to connect to ' + address)
-    return retData

onionr/communicatorutils/cooldownpeer.py

@@ -1,52 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    Select a random online peer in a communicator instance and have them "cool down"
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-from onionrutils import epoch
-def cooldown_peer(comm_inst):
-    '''Randomly add an online peer to cooldown, so we can connect a new one'''
-    onlinePeerAmount = len(comm_inst.onlinePeers)
-    minTime = 300
-    cooldownTime = 600
-    toCool = ''
-    tempConnectTimes = dict(comm_inst.connectTimes)
-
-    # Remove peers from cooldown that have been there long enough
-    tempCooldown = dict(comm_inst.cooldownPeer)
-    for peer in tempCooldown:
-        if (epoch.get_epoch() - tempCooldown[peer]) >= cooldownTime:
-            del comm_inst.cooldownPeer[peer]
-
-    # Cool down a peer, if we have max connections alive for long enough
-    if onlinePeerAmount >= comm_inst._core.config.get('peers.max_connect', 10, save = True):
-        finding = True
-
-        while finding:
-            try:
-                toCool = min(tempConnectTimes, key=tempConnectTimes.get)
-                if (epoch.get_epoch() - tempConnectTimes[toCool]) < minTime:
-                    del tempConnectTimes[toCool]
-                else:
-                    finding = False
-            except ValueError:
-                break
-        else:
-            comm_inst.removeOnlinePeer(toCool)
-            comm_inst.cooldownPeer[toCool] = epoch.get_epoch()
-
-    comm_inst.decrementThreadCount('cooldown_peer')

onionr/communicatorutils/daemonqueuehandler.py

@@ -1,56 +0,0 @@
-'''
-    Onionr - P2P Anonymous Storage Network
-
-    Handle daemon queue commands in the communicator
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import logger
-import onionrevents as events
-from onionrutils import localcommand
-def handle_daemon_commands(comm_inst):
-    cmd = comm_inst._core.daemonQueue()
-    response = ''
-    if cmd is not False:
-        events.event('daemon_command', onionr = comm_inst._core.onionrInst, data = {'cmd' : cmd})
-        if cmd[0] == 'shutdown':
-            comm_inst.shutdown = True
-        elif cmd[0] == 'announceNode':
-            if len(comm_inst.onlinePeers) > 0:
-                comm_inst.announce(cmd[1])
-            else:
-                logger.debug("No nodes connected. Will not introduce node.")
-        elif cmd[0] == 'runCheck': # deprecated
-            logger.debug('Status check; looks good.')
-            open(comm_inst._core.dataDir + '.runcheck', 'w+').close()
-        elif cmd[0] == 'connectedPeers':
-            response = '\n'.join(list(comm_inst.onlinePeers)).strip()
-            if response == '':
-                response = 'none'
-        elif cmd[0] == 'localCommand':
-            response = localcommand.local_command(comm_inst._core, cmd[1])
-        elif cmd[0] == 'pex':
-            for i in comm_inst.timers:
-                if i.timerFunction.__name__ == 'lookupAdders':
-                    i.count = (i.frequency - 1)
-        elif cmd[0] == 'uploadBlock':
-            comm_inst.blocksToUpload.append(cmd[1])
-
-        if cmd[0] not in ('', None):
-            if response != '':
-                localcommand.local_command(comm_inst._core, 'queueResponseAdd/' + cmd[4], post=True, postData={'data': response})
-        response = ''
-
-    comm_inst.decrementThreadCount('daemonCommands')

onionr/communicatorutils/deniableinserts.py

@@ -1,31 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    Use the communicator to insert fake mail messages
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import secrets
-from etc import onionrvalues
-def insert_deniable_block(comm_inst):
-    '''Insert a fake block in order to make it more difficult to track real blocks'''
-    fakePeer = ''
-    chance = 10
-    if secrets.randbelow(chance) == (chance - 1):
-        # This assumes on the libsodium primitives to have key-privacy
-        fakePeer = onionrvalues.DENIABLE_PEER_ADDRESS
-        data = secrets.token_hex(secrets.randbelow(1024) + 1)
-        comm_inst._core.insertBlock(data, header='pm', encryptType='asym', asymPeer=fakePeer, disableForward=True, meta={'subject': 'foo'})
-    comm_inst.decrementThreadCount('insert_deniable_block')

onionr/communicatorutils/downloadblocks.py

@@ -1,118 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    Download blocks using the communicator instance
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import communicator, onionrexceptions
-import logger, onionrpeers
-from onionrutils import blockmetadata, stringvalidators, validatemetadata
-
-def download_blocks_from_communicator(comm_inst):
-    assert isinstance(comm_inst, communicator.OnionrCommunicatorDaemon)
-    for blockHash in list(comm_inst.blockQueue):
-        if len(comm_inst.onlinePeers) == 0:
-            break
-        triedQueuePeers = [] # List of peers we've tried for a block
-        try:
-            blockPeers = list(comm_inst.blockQueue[blockHash])
-        except KeyError:
-            blockPeers = []
-        removeFromQueue = True
-        if comm_inst.shutdown or not comm_inst.isOnline:
-            # Exit loop if shutting down or offline
-            break
-        # Do not download blocks being downloaded or that are already saved (edge cases)
-        if blockHash in comm_inst.currentDownloading:
-            #logger.debug('Already downloading block %s...' % blockHash)
-            continue
-        if blockHash in comm_inst._core.getBlockList():
-            #logger.debug('Block %s is already saved.' % (blockHash,))
-            try:
-                del comm_inst.blockQueue[blockHash]
-            except KeyError:
-                pass
-            continue
-        if comm_inst._core._blacklist.inBlacklist(blockHash):
-            continue
-        if comm_inst._core.storage_counter.isFull():
-            break
-        comm_inst.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block
-        if len(blockPeers) == 0:
-            peerUsed = comm_inst.pickOnlinePeer()
-        else:
-            blockPeers = comm_inst._core._crypto.randomShuffle(blockPeers)
-            peerUsed = blockPeers.pop(0)
-
-        if not comm_inst.shutdown and peerUsed.strip() != '':
-            logger.info("Attempting to download %s from %s..." % (blockHash[:12], peerUsed))
-        content = comm_inst.peerAction(peerUsed, 'getdata/' + blockHash) # block content from random peer (includes metadata)
-        if content != False and len(content) > 0:
-            try:
-                content = content.encode()
-            except AttributeError:
-                pass
-
-            realHash = comm_inst._core._crypto.sha3Hash(content)
-            try:
-                realHash = realHash.decode() # bytes on some versions for some reason
-            except AttributeError:
-                pass
-            if realHash == blockHash:
-                content = content.decode() # decode here because sha3Hash needs bytes above
-                metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
-                metadata = metas[0]
-                if validatemetadata.validate_metadata(comm_inst._core, metadata, metas[2]): # check if metadata is valid, and verify nonce
-                    if comm_inst._core._crypto.verifyPow(content): # check if POW is enough/correct
-                        logger.info('Attempting to save block %s...' % blockHash[:12])
-                        try:
-                            comm_inst._core.setData(content)
-                        except onionrexceptions.DiskAllocationReached:
-                            logger.error('Reached disk allocation allowance, cannot save block %s.' % blockHash)
-                            removeFromQueue = False
-                        else:
-                            comm_inst._core.addToBlockDB(blockHash, dataSaved=True)
-                            blockmetadata.process_block_metadata(comm_inst._core, blockHash) # caches block metadata values to block database
-                    else:
-                        logger.warn('POW failed for block %s.' % blockHash)
-                else:
-                    if comm_inst._core._blacklist.inBlacklist(realHash):
-                        logger.warn('Block %s is blacklisted.' % (realHash,))
-                    else:
-                        logger.warn('Metadata for block %s is invalid.' % blockHash)
-                        comm_inst._core._blacklist.addToDB(blockHash)
-            else:
-                # if block didn't meet expected hash
-                tempHash = comm_inst._core._crypto.sha3Hash(content) # lazy hack, TODO use var
-                try:
-                    tempHash = tempHash.decode()
-                except AttributeError:
-                    pass
-                # Punish peer for sharing invalid block (not always malicious, but is bad regardless)
-                onionrpeers.PeerProfiles(peerUsed, comm_inst._core).addScore(-50)
-                if tempHash != 'ed55e34cb828232d6c14da0479709bfa10a0923dca2b380496e6b2ed4f7a0253':
-                    # Dumb hack for 404 response from peer. Don't log it if 404 since its likely not malicious or a critical error.
-                    logger.warn('Block hash validation failed for ' + blockHash + ' got ' + tempHash)
-                else:
-                    removeFromQueue = False # Don't remove from queue if 404
-            if removeFromQueue:
-                try:
-                    del comm_inst.blockQueue[blockHash] # remove from block queue both if success or false
-                    logger.info('%s blocks remaining in queue' % [len(comm_inst.blockQueue)], terminal=True)
-                except KeyError:
-                    pass
-        comm_inst.currentDownloading.remove(blockHash)
-    comm_inst.decrementThreadCount('getBlocks')

onionr/communicatorutils/housekeeping.py

@@ -1,60 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    Cleanup old Onionr blocks and forward secrecy keys using the communicator. Ran from a timer usually
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import sqlite3
-import logger
-from onionrusers import onionrusers
-from onionrutils import epoch
-def clean_old_blocks(comm_inst):
-    '''Delete old blocks if our disk allocation is full/near full, and also expired blocks'''
-
-    # Delete expired blocks
-    for bHash in comm_inst._core.getExpiredBlocks():
-        comm_inst._core._blacklist.addToDB(bHash)
-        comm_inst._core.removeBlock(bHash)
-        logger.info('Deleted block: %s' % (bHash,))
-
-    while comm_inst._core.storage_counter.isFull():
-        oldest = comm_inst._core.getBlockList()[0]
-        comm_inst._core._blacklist.addToDB(oldest)
-        comm_inst._core.removeBlock(oldest)
-        logger.info('Deleted block: %s' % (oldest,))
-
-    comm_inst.decrementThreadCount('clean_old_blocks')
-
-def clean_keys(comm_inst):
-    '''Delete expired forward secrecy keys'''
-    conn = sqlite3.connect(comm_inst._core.peerDB, timeout=10)
-    c = conn.cursor()
-    time = epoch.get_epoch()
-    deleteKeys = []
-
-    for entry in c.execute("SELECT * FROM forwardKeys WHERE expire <= ?", (time,)):
-        logger.debug('Forward key: %s' % entry[1])
-        deleteKeys.append(entry[1])
-
-    for key in deleteKeys:
-        logger.debug('Deleting forward key %s' % key)
-        c.execute("DELETE from forwardKeys where forwardKey = ?", (key,))
-    conn.commit()
-    conn.close()
-
-    onionrusers.deleteExpiredKeys(comm_inst._core)
-
-    comm_inst.decrementThreadCount('clean_keys')

onionr/communicatorutils/lookupadders.py

@@ -1,49 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    Lookup new peer transport addresses using the communicator
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import logger
-from onionrutils import stringvalidators
-
-def lookup_new_peer_transports_with_communicator(comm_inst):
-    logger.info('Looking up new addresses...')
-    tryAmount = 1
-    newPeers = []
-    for i in range(tryAmount):
-        # Download new peer address list from random online peers
-        if len(newPeers) > 10000:
-            # Don't get new peers if we have too many queued up
-            break
-        peer = comm_inst.pickOnlinePeer()
-        newAdders = comm_inst.peerAction(peer, action='pex')
-        try:
-            newPeers = newAdders.split(',')
-        except AttributeError:
-            pass
-    else:
-        # Validate new peers are good format and not already in queue
-        invalid = []
-        for x in newPeers:
-            x = x.strip()
-            if not stringvalidators.validate_transport(x) or x in comm_inst.newPeers or x == comm_inst._core.hsAddress:
-                # avoid adding if its our address
-                invalid.append(x)
-        for x in invalid:
-            newPeers.remove(x)
-        comm_inst.newPeers.extend(newPeers)
-    comm_inst.decrementThreadCount('lookupAdders')

onionr/communicatorutils/lookupblocks.py

@@ -1,88 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    Lookup new blocks with the communicator using a random connected peer
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import logger, onionrproofs
-from onionrutils import stringvalidators, epoch
-
-def lookup_blocks_from_communicator(comm_inst):
-    logger.info('Looking up new blocks...')
-    tryAmount = 2
-    newBlocks = ''
-    existingBlocks = comm_inst._core.getBlockList()
-    triedPeers = [] # list of peers we've tried this time around
-    maxBacklog = 1560 # Max amount of *new* block hashes to have already in queue, to avoid memory exhaustion
-    lastLookupTime = 0 # Last time we looked up a particular peer's list
-    new_block_count = 0
-    for i in range(tryAmount):
-        listLookupCommand = 'getblocklist' # This is defined here to reset it each time
-        if len(comm_inst.blockQueue) >= maxBacklog:
-            break
-        if not comm_inst.isOnline:
-            break
-        # check if disk allocation is used
-        if comm_inst._core.storage_counter.isFull():
-            logger.debug('Not looking up new blocks due to maximum amount of allowed disk space used')
-            break
-        peer = comm_inst.pickOnlinePeer() # select random online peer
-        # if we've already tried all the online peers this time around, stop
-        if peer in triedPeers:
-            if len(comm_inst.onlinePeers) == len(triedPeers):
-                break
-            else:
-                continue
-        triedPeers.append(peer)
-
-        # Get the last time we looked up a peer's stamp to only fetch blocks since then.
-        # Saved in memory only for privacy reasons
-        try:
-            lastLookupTime = comm_inst.dbTimestamps[peer]
-        except KeyError:
-            lastLookupTime = 0
-        else:
-            listLookupCommand += '?date=%s' % (lastLookupTime,)
-        try:
-            newBlocks = comm_inst.peerAction(peer, listLookupCommand) # get list of new block hashes
-        except Exception as error:
-            logger.warn('Could not get new blocks from %s.' % peer, error = error)
-            newBlocks = False
-        else:
-            comm_inst.dbTimestamps[peer] = epoch.get_rounded_epoch(roundS=60)
-        if newBlocks != False:
-            # if request was a success
-            for i in newBlocks.split('\n'):
-                if stringvalidators.validate_hash(i):
-                    # if newline seperated string is valid hash
-                    if not i in existingBlocks:
-                        # if block does not exist on disk and is not already in block queue
-                        if i not in comm_inst.blockQueue:
-                            if onionrproofs.hashMeetsDifficulty(i) and not comm_inst._core._blacklist.inBlacklist(i):
-                                if len(comm_inst.blockQueue) <= 1000000:
-                                    comm_inst.blockQueue[i] = [peer] # add blocks to download queue
-                                    new_block_count += 1
-                        else:
-                            if peer not in comm_inst.blockQueue[i]:
-                                if len(comm_inst.blockQueue[i]) < 10:
-                                    comm_inst.blockQueue[i].append(peer)
-    if new_block_count > 0:
-        block_string = ""
-        if new_block_count > 1:
-            block_string = "s"
-        logger.info('Discovered %s new block%s' % (new_block_count, block_string), terminal=True)
-    comm_inst.decrementThreadCount('lookupBlocks')
-    return

onionr/communicatorutils/netcheck.py

@@ -1,41 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    Determine if our node is able to use Tor based on the status of a communicator instance
-    and the result of pinging onion http servers
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import logger
-from utils import netutils
-from onionrutils import localcommand, epoch
-def net_check(comm_inst):
-    '''Check if we are connected to the internet or not when we can't connect to any peers'''
-    rec = False # for detecting if we have received incoming connections recently
-    c = comm_inst._core
-    if len(comm_inst.onlinePeers) == 0:
-        try:
-            if (epoch.get_epoch() - int(localcommand.local_command(c, '/lastconnect'))) <= 60:
-                comm_inst.isOnline = True
-                rec = True
-        except ValueError:
-            pass
-        if not rec and not netutils.checkNetwork(c, torPort=comm_inst.proxyPort):
-            if not comm_inst.shutdown:
-                logger.warn('Network check failed, are you connected to the Internet, and is Tor working?')
-            comm_inst.isOnline = False
-        else:
-            comm_inst.isOnline = True
-    comm_inst.decrementThreadCount('net_check')

onionr/communicatorutils/onionrcommunicatortimers.py

@@ -1,63 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    This file contains timer control for the communicator
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import threading, onionrexceptions, logger
-class OnionrCommunicatorTimers:
-    def __init__(self, daemonInstance, timerFunction, frequency, makeThread=True, threadAmount=1, maxThreads=5, requiresPeer=False, myArgs=[]):
-        self.timerFunction = timerFunction
-        self.frequency = frequency
-        self.threadAmount = threadAmount
-        self.makeThread = makeThread
-        self.requiresPeer = requiresPeer
-        self.daemonInstance = daemonInstance
-        self.maxThreads = maxThreads
-        self._core = self.daemonInstance._core
-        self.args = myArgs
-
-        self.daemonInstance.timers.append(self)
-        self.count = 0
-
-    def processTimer(self):
-
-        # mark how many instances of a thread we have (decremented at thread end)
-        try:
-            self.daemonInstance.threadCounts[self.timerFunction.__name__]
-        except KeyError:
-            self.daemonInstance.threadCounts[self.timerFunction.__name__] = 0
-
-        # execute thread if it is time, and we are not missing *required* online peer
-        if self.count == self.frequency and not self.daemonInstance.shutdown:
-            try:
-                if self.requiresPeer and len(self.daemonInstance.onlinePeers) == 0:
-                    raise onionrexceptions.OnlinePeerNeeded
-            except onionrexceptions.OnlinePeerNeeded:
-                pass
-            else:
-                if self.makeThread:
-                    for i in range(self.threadAmount):
-                        if self.daemonInstance.threadCounts[self.timerFunction.__name__] >= self.maxThreads:
-                            logger.debug('%s is currently using the maximum number of threads, not starting another.' % self.timerFunction.__name__)
-                        else:
-                            self.daemonInstance.threadCounts[self.timerFunction.__name__] += 1
-                            newThread = threading.Thread(target=self.timerFunction, args=self.args, daemon=True)
-                            newThread.start()
-                else:
-                    self.timerFunction()
-            self.count = -1 # negative 1 because its incremented at bottom
-        self.count += 1

onionr/communicatorutils/proxypicker.py

@@ -1,26 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    Just picks a proxy to use based on a peer's address
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-
-def pick_proxy(peer_address):
-    if peer_address.endswith('.onion'):
-        return 'tor'
-    elif peer_address.endswith('.i2p'):
-        return 'i2p'
-    raise ValueError("Peer address was not string ending with acceptable value")

onionr/communicatorutils/servicecreator.py

@@ -1,41 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    Creates an onionr direct connection service by scanning all connection blocks
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import communicator, onionrblockapi
-from onionrutils import stringvalidators
-
-def service_creator(daemon):
-    assert isinstance(daemon, communicator.OnionrCommunicatorDaemon)
-    core = daemon._core
-    utils = core._utils
-    
-    # Find socket connection blocks
-    # TODO cache blocks and only look at recently received ones
-    con_blocks = core.getBlocksByType('con')
-    for b in con_blocks:
-        if not b in daemon.active_services:
-            bl = onionrblockapi.Block(b, core=core, decrypt=True)
-            bs = utils.bytesToStr(bl.bcontent) + '.onion'
-            if stringvalidators.validate_pub_key(bl.signer) and stringvalidators.validate_transport(bs):
-                signer = utils.bytesToStr(bl.signer)
-                daemon.active_services.append(b)
-                daemon.active_services.append(signer)
-                daemon.services.create_server(signer, bs)
-    
-    daemon.decrementThreadCount('service_creator')

onionr/communicatorutils/uploadblocks.py

@@ -1,54 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    Upload blocks in the upload queue to peers from the communicator
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import logger
-from communicatorutils import proxypicker
-import onionrblockapi as block
-from onionrutils import localcommand, stringvalidators, basicrequests
-
-def upload_blocks_from_communicator(comm_inst):
-    # when inserting a block, we try to upload it to a few peers to add some deniability
-    triedPeers = []
-    finishedUploads = []
-    core = comm_inst._core
-    comm_inst.blocksToUpload = core._crypto.randomShuffle(comm_inst.blocksToUpload)
-    if len(comm_inst.blocksToUpload) != 0:
-        for bl in comm_inst.blocksToUpload:
-            if not stringvalidators.validate_hash(bl):
-                logger.warn('Requested to upload invalid block')
-                comm_inst.decrementThreadCount('uploadBlock')
-                return
-            for i in range(min(len(comm_inst.onlinePeers), 6)):
-                peer = comm_inst.pickOnlinePeer()
-                if peer in triedPeers:
-                    continue
-                triedPeers.append(peer)
-                url = 'http://' + peer + '/upload'
-                data = {'block': block.Block(bl).getRaw()}
-                proxyType = proxypicker.pick_proxy(peer)
-                logger.info("Uploading block to " + peer, terminal=True)
-                if not basicrequests.do_post_request(core, url, data=data, proxyType=proxyType) == False:
-                    localcommand.local_command(core, 'waitforshare/' + bl, post=True)
-                    finishedUploads.append(bl)
-    for x in finishedUploads:
-        try:
-            comm_inst.blocksToUpload.remove(x)
-        except ValueError:
-            pass
-    comm_inst.decrementThreadCount('uploadBlock')

onionr/config.py

@@ -1,150 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    This file deals with configuration management.
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-
-import os, json, logger
-
-# set data dir
-dataDir = os.environ.get('ONIONR_HOME', os.environ.get('DATA_DIR', 'data/'))
-if not dataDir.endswith('/'):
-    dataDir += '/'
-
-_configfile = os.path.abspath(dataDir + 'config.json')
-_config = {}
-
-def get(key, default = None, save = False):
-    '''
-        Gets the key from configuration, or returns `default`
-    '''
-
-    key = str(key).split('.')
-    data = _config
-
-    last = key.pop()
-
-    for item in key:
-        if (not item in data) or (not type(data[item]) == dict):
-            return default
-        data = data[item]
-
-    if not last in data:
-        if save:
-            set(key, default, savefile = True)
-        return default
-
-    return data[last]
-
-def set(key, value = None, savefile = False):
-    '''
-        Sets the key in configuration to `value`
-    '''
-
-    global _config
-
-    key = str(key).split('.')
-    data = _config
-
-    last = key.pop()
-
-    for item in key:
-        if (not item in data) or (not type(data[item]) == dict):
-            data[item] = dict()
-        data = data[item]
-
-    if value is None:
-        del data[last]
-    else:
-        data[last] = value
-
-    if savefile:
-        save()
-
-def is_set(key):
-    key = str(key).split('.')
-    data = _config
-
-    last = key.pop()
-
-    for item in key:
-        if (not item in data) or (not type(data[item]) == dict):
-            return False
-        data = data[item]
-
-    if not last in data:
-        return False
-
-    return True
-
-def check():
-    '''
-        Checks if the configuration file exists, creates it if not
-    '''
-
-    if not os.path.exists(os.path.dirname(get_config_file())):
-        os.makedirs(os.path.dirname(get_config_file()))
-
-def save():
-    '''
-        Saves the configuration data to the configuration file
-    '''
-
-    check()
-    try:
-        with open(get_config_file(), 'w', encoding="utf8") as configfile:
-            json.dump(get_config(), configfile, indent=2)
-    except json.JSONDecodeError:
-        logger.warn('Failed to write to configuration file.')
-
-def reload():
-    '''
-        Reloads the configuration data in memory from the file
-    '''
-    check()
-    try:
-        with open(get_config_file(), 'r', encoding="utf8") as configfile:
-            set_config(json.loads(configfile.read()))
-    except:
-        pass
-        #logger.debug('Failed to parse configuration file.')
-
-def get_config():
-    '''
-        Gets the entire configuration as an array
-    '''
-    return _config
-
-def set_config(config):
-    '''
-        Sets the configuration to the array in arguments
-    '''
-    global _config
-    _config = config
-
-def get_config_file():
-    '''
-        Returns the absolute path to the configuration file
-    '''
-    return _configfile
-
-def set_config_file(configfile):
-    '''
-        Sets the path to the configuration file
-    '''
-    global _configfile
-    _configfile = os.abs.abspath(configfile)

onionr/core.py

@@ -1,462 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    Core Onionr library, useful for external programs. Handles peer & data processing
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import os, sys, json
-import logger, netcontroller, config
-from onionrblockapi import Block
-import coredb
-import deadsimplekv as simplekv
-import onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions
-import onionrblacklist
-from onionrusers import onionrusers
-from onionrstorage import removeblock, setdata
-import dbcreator, onionrstorage, serializeddata, subprocesspow
-from etc import onionrvalues, powchoice
-from onionrutils import localcommand, stringvalidators, bytesconverter, epoch
-from onionrutils import blockmetadata
-import storagecounter
-
-class Core:
-    def __init__(self, torPort=0):
-        '''
-            Initialize Core Onionr library
-        '''
-        # set data dir
-        self.dataDir = os.environ.get('ONIONR_HOME', os.environ.get('DATA_DIR', 'data/'))
-        if not self.dataDir.endswith('/'):
-            self.dataDir += '/'
-
-        try:
-            self.usageFile = self.dataDir + 'disk-usage.txt'
-            self.config = config
-            self.maxBlockSize = 10000000 # max block size in bytes
-
-            self.onionrInst = None
-            self.queueDB = self.dataDir + 'queue.db'
-            self.peerDB = self.dataDir + 'peers.db'
-            self.blockDB = self.dataDir + 'blocks.db'
-            self.blockDataLocation = self.dataDir + 'blocks/'
-            self.blockDataDB = self.blockDataLocation + 'block-data.db'
-            self.publicApiHostFile = self.dataDir + 'public-host.txt'
-            self.privateApiHostFile = self.dataDir + 'private-host.txt'
-            self.addressDB = self.dataDir + 'address.db'
-            self.hsAddress = ''
-            self.i2pAddress = config.get('i2p.own_addr', None)
-            self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt'
-            self.bootstrapList = []
-            self.requirements = onionrvalues.OnionrValues()
-            self.torPort = torPort
-            self.dataNonceFile = self.dataDir + 'block-nonces.dat'
-            self.dbCreate = dbcreator.DBCreator(self)
-            self.forwardKeysFile = self.dataDir + 'forward-keys.db'
-            self.keyStore = simplekv.DeadSimpleKV(self.dataDir + 'cachedstorage.dat', refresh_seconds=5)
-            self.storage_counter = storagecounter.StorageCounter(self)
-            
-            # Socket data, defined here because of multithreading constraints with gevent
-            self.killSockets = False
-            self.startSocket = {}
-            self.socketServerConnData = {}
-            self.socketReasons = {}
-            self.socketServerResponseData = {}
-
-            if not os.path.exists(self.dataDir):
-                os.mkdir(self.dataDir)
-            if not os.path.exists(self.dataDir + 'blocks/'):
-                os.mkdir(self.dataDir + 'blocks/')
-            if not os.path.exists(self.blockDB):
-                self.createBlockDB()
-            if not os.path.exists(self.forwardKeysFile):
-                self.dbCreate.createForwardKeyDB()
-            if not os.path.exists(self.peerDB):
-                self.createPeerDB()
-            if not os.path.exists(self.addressDB):
-                self.createAddressDB()
-
-            if os.path.exists(self.dataDir + '/hs/hostname'):
-                with open(self.dataDir + '/hs/hostname', 'r') as hs:
-                    self.hsAddress = hs.read().strip()
-
-            # Load bootstrap address list
-            if os.path.exists(self.bootstrapFileLocation):
-                with open(self.bootstrapFileLocation, 'r') as bootstrap:
-                    bootstrap = bootstrap.read()
-                for i in bootstrap.split('\n'):
-                    self.bootstrapList.append(i)
-            else:
-                logger.warn('Warning: address bootstrap file not found ' + self.bootstrapFileLocation)
-
-            self.use_subprocess = powchoice.use_subprocess(self)
-            # Initialize the crypto object
-            self._crypto = onionrcrypto.OnionrCrypto(self)
-            self._blacklist = onionrblacklist.OnionrBlackList(self)
-            self.serializer = serializeddata.SerializedData(self)
-
-        except Exception as error:
-            logger.error('Failed to initialize core Onionr library.', error=error, terminal=True)
-            logger.fatal('Cannot recover from error.', terminal=True)
-            sys.exit(1)
-        return
-
-    def refreshFirstStartVars(self):
-        '''
-            Hack to refresh some vars which may not be set on first start
-        '''
-        if os.path.exists(self.dataDir + '/hs/hostname'):
-            with open(self.dataDir + '/hs/hostname', 'r') as hs:
-                self.hsAddress = hs.read().strip()
-
-    def addPeer(self, peerID, name=''):
-        '''
-            Adds a public key to the key database (misleading function name)
-        '''
-        return coredb.keydb.addkeys.add_peer(self, peerID, name)
-
-    def addAddress(self, address):
-        '''
-            Add an address to the address database (only tor currently)
-        '''
-        return coredb.keydb.addkeys.add_address(self, address)
-
-    def removeAddress(self, address):
-        '''
-            Remove an address from the address database
-        '''
-        return coredb.keydb.removekeys.remove_address(self, address)
-
-    def removeBlock(self, block):
-        '''
-            remove a block from this node (does not automatically blacklist)
-
-            **You may want blacklist.addToDB(blockHash)
-        '''
-        removeblock.remove_block(self, block)
-
-    def createAddressDB(self):
-        '''
-            Generate the address database
-        '''
-        self.dbCreate.createAddressDB()
-
-    def createPeerDB(self):
-        '''
-            Generate the peer sqlite3 database and populate it with the peers table.
-        '''
-        self.dbCreate.createPeerDB()
-
-    def createBlockDB(self):
-        '''
-            Create a database for blocks
-        '''
-        self.dbCreate.createBlockDB()
-
-    def addToBlockDB(self, newHash, selfInsert=False, dataSaved=False):
-        '''
-            Add a hash value to the block db
-
-            Should be in hex format!
-        '''
-        coredb.blockmetadb.add.add_to_block_DB(self, newHash, selfInsert, dataSaved)
-
-    def setData(self, data):
-        '''
-            Set the data assciated with a hash
-        '''
-        return onionrstorage.setdata.set_data(self, data)
-
-    def getData(self, hash):
-        '''
-            Simply return the data associated to a hash
-        '''
-        return onionrstorage.getData(self, hash)
-
-    def daemonQueue(self):
-        '''
-            Gives commands to the communication proccess/daemon by reading an sqlite3 database
-
-            This function intended to be used by the client. Queue to exchange data between "client" and server.
-        '''
-        return coredb.daemonqueue.daemon_queue(self)
-
-    def daemonQueueAdd(self, command, data='', responseID=''):
-        '''
-            Add a command to the daemon queue, used by the communication daemon (communicator.py)
-        '''
-        return coredb.daemonqueue.daemon_queue_add(self, command, data, responseID)
-
-    def daemonQueueGetResponse(self, responseID=''):
-        '''
-            Get a response sent by communicator to the API, by requesting to the API
-        '''
-        return coredb.daemonqueue.daemon_queue_get_response(self, responseID)
-
-    def clearDaemonQueue(self):
-        '''
-            Clear the daemon queue (somewhat dangerous)
-        '''
-        return coredb.daemonqueue.clear_daemon_queue(self)
-
-    def listAdders(self, randomOrder=True, i2p=True, recent=0):
-        '''
-            Return a list of addresses
-        '''
-        return coredb.keydb.listkeys.list_adders(self, randomOrder, i2p, recent)
-
-    def listPeers(self, randomOrder=True, getPow=False, trust=0):
-        '''
-            Return a list of public keys (misleading function name)
-
-            randomOrder determines if the list should be in a random order
-            trust sets the minimum trust to list
-        '''
-        return coredb.keydb.listkeys.list_peers(self, randomOrder, getPow, trust)
-
-    def getPeerInfo(self, peer, info):
-        '''
-            Get info about a peer from their database entry
-
-            id text             0
-            name text,          1
-            adders text,        2
-            dateSeen not null,  3
-            trust int           4
-            hashID text         5
-        '''
-        return coredb.keydb.userinfo.get_user_info(self, peer, info)
-
-    def setPeerInfo(self, peer, key, data):
-        '''
-            Update a peer for a key
-        '''
-        return coredb.keydb.userinfo.set_peer_info(self, peer, key, data)
-
-    def getAddressInfo(self, address, info):
-        '''
-            Get info about an address from its database entry
-
-            address text, 0
-            type int, 1
-            knownPeer text, 2
-            speed int, 3
-            success int, 4
-            powValue    5
-            failure int 6
-            lastConnect 7
-            trust       8
-            introduced  9
-        '''
-        return coredb.keydb.transportinfo.get_address_info(self, address, info)
-
-    def setAddressInfo(self, address, key, data):
-        '''
-            Update an address for a key
-        '''
-        return coredb.keydb.transportinfo.set_address_info(self, address, key, data)
-
-    def getBlockList(self, dateRec = None, unsaved = False):
-        '''
-            Get list of our blocks
-        '''
-        return coredb.blockmetadb.get_block_list(self, dateRec, unsaved)
-
-    def getBlockDate(self, blockHash):
-        '''
-            Returns the date a block was received
-        '''
-        return coredb.blockmetadb.get_block_date(self, blockHash)
-
-    def getBlocksByType(self, blockType, orderDate=True):
-        '''
-            Returns a list of blocks by the type
-        '''
-        return coredb.blockmetadb.get_blocks_by_type(self, blockType, orderDate)
-
-    def getExpiredBlocks(self):
-        '''Returns a list of expired blocks'''
-        return coredb.blockmetadb.expiredblocks.get_expired_blocks(self)
-
-    def updateBlockInfo(self, hash, key, data):
-        '''
-            sets info associated with a block
-
-            hash         - the hash of a block
-            dateReceived - the date the block was recieved, not necessarily when it was created
-            decrypted    - if we can successfully decrypt the block (does not describe its current state)
-            dataType     - data type of the block
-            dataFound    - if the data has been found for the block
-            dataSaved    - if the data has been saved for the block
-            sig    - optional signature by the author (not optional if author is specified)
-            author       - multi-round partial sha3-256 hash of authors public key
-            dateClaimed  - timestamp claimed inside the block, only as trustworthy as the block author is
-            expire       - expire date for a block
-        '''
-        return coredb.blockmetadb.updateblockinfo.update_block_info(self, hash, key, data)
-
-    def insertBlock(self, data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = {}, expire=None, disableForward=False):
-        '''
-            Inserts a block into the network
-            encryptType must be specified to encrypt a block
-        '''
-        allocationReachedMessage = 'Cannot insert block, disk allocation reached.'
-        if self.storage_counter.isFull():
-            logger.error(allocationReachedMessage)
-            return False
-        retData = False
-
-        if type(data) is None:
-            raise ValueError('Data cannot be none')
-
-        createTime = epoch.get_epoch()
-
-        dataNonce = bytesconverter.bytes_to_str(self._crypto.sha3Hash(data))
-        try:
-            with open(self.dataNonceFile, 'r') as nonces:
-                if dataNonce in nonces:
-                    return retData
-        except FileNotFoundError:
-            pass
-        # record nonce
-        with open(self.dataNonceFile, 'a') as nonceFile:
-            nonceFile.write(dataNonce + '\n')
-
-        if type(data) is bytes:
-            data = data.decode()
-        data = str(data)
-        plaintext = data
-        plaintextMeta = {}
-        plaintextPeer = asymPeer
-
-        retData = ''
-        signature = ''
-        signer = ''
-        metadata = {}
-        # metadata is full block metadata, meta is internal, user specified metadata
-
-        # only use header if not set in provided meta
-
-        meta['type'] = str(header)
-
-        if encryptType in ('asym', 'sym', ''):
-            metadata['encryptType'] = encryptType
-        else:
-            raise onionrexceptions.InvalidMetadata('encryptType must be asym or sym, or blank')
-
-        try:
-            data = data.encode()
-        except AttributeError:
-            pass
-
-        if encryptType == 'asym':
-            meta['rply'] = createTime # Duplicate the time in encrypted messages to prevent replays
-            if not disableForward and sign and asymPeer != self._crypto.pubKey:
-                try:
-                    forwardEncrypted = onionrusers.OnionrUser(self, asymPeer).forwardEncrypt(data)
-                    data = forwardEncrypted[0]
-                    meta['forwardEnc'] = True
-                    expire = forwardEncrypted[2] # Expire time of key. no sense keeping block after that
-                except onionrexceptions.InvalidPubkey:
-                    pass
-                    #onionrusers.OnionrUser(self, asymPeer).generateForwardKey()
-                fsKey = onionrusers.OnionrUser(self, asymPeer).generateForwardKey()
-                #fsKey = onionrusers.OnionrUser(self, asymPeer).getGeneratedForwardKeys().reverse()
-                meta['newFSKey'] = fsKey
-        jsonMeta = json.dumps(meta)
-        plaintextMeta = jsonMeta
-        if sign:
-            signature = self._crypto.edSign(jsonMeta.encode() + data, key=self._crypto.privKey, encodeResult=True)
-            signer = self._crypto.pubKey
-
-        if len(jsonMeta) > 1000:
-            raise onionrexceptions.InvalidMetadata('meta in json encoded form must not exceed 1000 bytes')
-
-        user = onionrusers.OnionrUser(self, symKey)
-
-        # encrypt block metadata/sig/content
-        if encryptType == 'sym':
-
-            if len(symKey) < self.requirements.passwordLength:
-                raise onionrexceptions.SecurityError('Weak encryption key')
-            jsonMeta = self._crypto.symmetricEncrypt(jsonMeta, key=symKey, returnEncoded=True).decode()
-            data = self._crypto.symmetricEncrypt(data, key=symKey, returnEncoded=True).decode()
-            signature = self._crypto.symmetricEncrypt(signature, key=symKey, returnEncoded=True).decode()
-            signer = self._crypto.symmetricEncrypt(signer, key=symKey, returnEncoded=True).decode()
-        elif encryptType == 'asym':
-            if stringvalidators.validate_pub_key(asymPeer):
-                # Encrypt block data with forward secrecy key first, but not meta
-                jsonMeta = json.dumps(meta)
-                jsonMeta = self._crypto.pubKeyEncrypt(jsonMeta, asymPeer, encodedData=True).decode()
-                data = self._crypto.pubKeyEncrypt(data, asymPeer, encodedData=True).decode()
-                signature = self._crypto.pubKeyEncrypt(signature, asymPeer, encodedData=True).decode()
-                signer = self._crypto.pubKeyEncrypt(signer, asymPeer, encodedData=True).decode()
-                try:
-                    onionrusers.OnionrUser(self, asymPeer, saveUser=True)
-                except ValueError:
-                    # if peer is already known
-                    pass
-            else:
-                raise onionrexceptions.InvalidPubkey(asymPeer + ' is not a valid base32 encoded ed25519 key')
-
-        # compile metadata
-        metadata['meta'] = jsonMeta
-        metadata['sig'] = signature
-        metadata['signer'] = signer
-        metadata['time'] = createTime
-
-        # ensure expire is integer and of sane length
-        if type(expire) is not type(None):
-            assert len(str(int(expire))) < 14
-            metadata['expire'] = expire
-
-        # send block data (and metadata) to POW module to get tokenized block data
-        if self.use_subprocess:
-            payload = subprocesspow.SubprocessPOW(data, metadata, self).start()
-        else:
-            payload = onionrproofs.POW(metadata, data).waitForResult()
-        if payload != False:
-            try:
-                retData = self.setData(payload)
-            except onionrexceptions.DiskAllocationReached:
-                logger.error(allocationReachedMessage)
-                retData = False
-            else:
-                # Tell the api server through localCommand to wait for the daemon to upload this block to make statistical analysis more difficult
-                if localcommand.local_command(self, '/ping', maxWait=10) == 'pong!':
-                    if self.config.get('general.security_level', 1) == 0:
-                        localcommand.local_command(self, '/waitforshare/' + retData, post=True, maxWait=5)
-                    self.daemonQueueAdd('uploadBlock', retData)
-                else:
-                    pass
-                self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
-                blockmetadata.process_block_metadata(self, retData)
-
-        if retData != False:
-            if plaintextPeer == onionrvalues.DENIABLE_PEER_ADDRESS:
-                events.event('insertdeniable', {'content': plaintext, 'meta': plaintextMeta, 'hash': retData, 'peer': bytesconverter.bytes_to_str(asymPeer)}, onionr = self.onionrInst, threaded = True)
-            else:
-                events.event('insertblock', {'content': plaintext, 'meta': plaintextMeta, 'hash': retData, 'peer': bytesconverter.bytes_to_str(asymPeer)}, onionr = self.onionrInst, threaded = True)
-        return retData
-
-    def introduceNode(self):
-        '''
-            Introduces our node into the network by telling X many nodes our HS address
-        '''
-        if localcommand.local_command(self, '/ping', maxWait=10) == 'pong!':
-            self.daemonQueueAdd('announceNode')
-            logger.info('Introduction command will be processed.', terminal=True)
-        else:
-            logger.warn('No running node detected. Cannot introduce.', terminal=True)

onionr/coredb/__init__.py

@@ -1 +0,0 @@
-from . import keydb, blockmetadb, daemonqueue

onionr/coredb/blockmetadb/__init__.py

@@ -1,77 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    This module works with information relating to blocks stored on the node
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import sqlite3
-from . import expiredblocks, updateblockinfo, add
-def get_block_list(core_inst, dateRec = None, unsaved = False):
-    '''
-        Get list of our blocks
-    '''
-    if dateRec == None:
-        dateRec = 0
-
-    conn = sqlite3.connect(core_inst.blockDB, timeout=30)
-    c = conn.cursor()
-
-    execute = 'SELECT hash FROM hashes WHERE dateReceived >= ? ORDER BY dateReceived ASC;'
-    args = (dateRec,)
-    rows = list()
-    for row in c.execute(execute, args):
-        for i in row:
-            rows.append(i)
-    conn.close()
-    return rows
-
-def get_block_date(core_inst, blockHash):
-    '''
-        Returns the date a block was received
-    '''
-
-    conn = sqlite3.connect(core_inst.blockDB, timeout=30)
-    c = conn.cursor()
-
-    execute = 'SELECT dateReceived FROM hashes WHERE hash=?;'
-    args = (blockHash,)
-    for row in c.execute(execute, args):
-        for i in row:
-            return int(i)
-    conn.close()
-    return None
-
-def get_blocks_by_type(core_inst, blockType, orderDate=True):
-    '''
-        Returns a list of blocks by the type
-    '''
-
-    conn = sqlite3.connect(core_inst.blockDB, timeout=30)
-    c = conn.cursor()
-
-    if orderDate:
-        execute = 'SELECT hash FROM hashes WHERE dataType=? ORDER BY dateReceived;'
-    else:
-        execute = 'SELECT hash FROM hashes WHERE dataType=?;'
-
-    args = (blockType,)
-    rows = list()
-
-    for row in c.execute(execute, args):
-        for i in row:
-            rows.append(i)
-    conn.close()
-    return rows

onionr/coredb/blockmetadb/add.py

@@ -1,43 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    Add an entry to the block metadata database
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import os, sqlite3
-from onionrutils import epoch, blockmetadata
-def add_to_block_DB(core_inst, newHash, selfInsert=False, dataSaved=False):
-    '''
-        Add a hash value to the block db
-
-        Should be in hex format!
-    '''
-
-    if not os.path.exists(core_inst.blockDB):
-        raise Exception('Block db does not exist')
-    if blockmetadata.has_block(core_inst, newHash):
-        return
-    conn = sqlite3.connect(core_inst.blockDB, timeout=30)
-    c = conn.cursor()
-    currentTime = epoch.get_epoch() + core_inst._crypto.secrets.randbelow(301)
-    if selfInsert or dataSaved:
-        selfInsert = 1
-    else:
-        selfInsert = 0
-    data = (newHash, currentTime, '', selfInsert)
-    c.execute('INSERT INTO hashes (hash, dateReceived, dataType, dataSaved) VALUES(?, ?, ?, ?);', data)
-    conn.commit()
-    conn.close()

onionr/coredb/blockmetadb/expiredblocks.py

@@ -1,35 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    Get a list of expired blocks still stored
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import sqlite3
-from onionrutils import epoch
-def get_expired_blocks(core_inst):
-    '''Returns a list of expired blocks'''
-    conn = sqlite3.connect(core_inst.blockDB, timeout=30)
-    c = conn.cursor()
-    date = int(epoch.get_epoch())
-
-    execute = 'SELECT hash FROM hashes WHERE expire <= %s ORDER BY dateReceived;' % (date,)
-
-    rows = list()
-    for row in c.execute(execute):
-        for i in row:
-            rows.append(i)
-    conn.close()
-    return rows

onionr/coredb/blockmetadb/updateblockinfo.py

@@ -1,33 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    Update block information in the metadata database by a field name
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-
-import sqlite3
-def update_block_info(core_inst, hash, key, data):
-    if key not in ('dateReceived', 'decrypted', 'dataType', 'dataFound', 'dataSaved', 'sig', 'author', 'dateClaimed', 'expire'):
-        return False
-
-    conn = sqlite3.connect(core_inst.blockDB, timeout=30)
-    c = conn.cursor()
-    args = (data, hash)
-    c.execute("UPDATE hashes SET " + key + " = ? where hash = ?;", args)
-    conn.commit()
-    conn.close()
-
-    return True

onionr/coredb/daemonqueue/__init__.py

@@ -1,97 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    Write and read the daemon queue, which is how messages are passed into the onionr daemon in a more
-    direct way than the http api
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import sqlite3, os
-import onionrevents as events
-from onionrutils import localcommand, epoch
-
-def daemon_queue(core_inst):
-    '''
-        Gives commands to the communication proccess/daemon by reading an sqlite3 database
-
-        This function intended to be used by the client. Queue to exchange data between "client" and server.
-    '''
-
-    retData = False
-    if not os.path.exists(core_inst.queueDB):
-        core_inst.dbCreate.createDaemonDB()
-    else:
-        conn = sqlite3.connect(core_inst.queueDB, timeout=30)
-        c = conn.cursor()
-        try:
-            for row in c.execute('SELECT command, data, date, min(ID), responseID FROM commands group by id'):
-                retData = row
-                break
-        except sqlite3.OperationalError:
-            core_inst.dbCreate.createDaemonDB()
-        else:
-            if retData != False:
-                c.execute('DELETE FROM commands WHERE id=?;', (retData[3],))
-        conn.commit()
-        conn.close()
-
-    events.event('queue_pop', data = {'data': retData}, onionr = core_inst.onionrInst)
-
-    return retData
-
-def daemon_queue_add(core_inst, command, data='', responseID=''):
-    '''
-        Add a command to the daemon queue, used by the communication daemon (communicator.py)
-    '''
-
-    retData = True
-
-    date = epoch.get_epoch()
-    conn = sqlite3.connect(core_inst.queueDB, timeout=30)
-    c = conn.cursor()
-    t = (command, data, date, responseID)
-    try:
-        c.execute('INSERT INTO commands (command, data, date, responseID) VALUES(?, ?, ?, ?)', t)
-        conn.commit()
-    except sqlite3.OperationalError:
-        retData = False
-        core_inst.daemonQueue()
-    events.event('queue_push', data = {'command': command, 'data': data}, onionr = core_inst.onionrInst)
-    conn.close()
-    return retData
-
-def daemon_queue_get_response(core_inst, responseID=''):
-    '''
-        Get a response sent by communicator to the API, by requesting to the API
-    '''
-    assert len(responseID) > 0
-    resp = localcommand.local_command(core_inst, 'queueResponse/' + responseID)
-    return resp
-
-def clear_daemon_queue(core_inst):
-    '''
-        Clear the daemon queue (somewhat dangerous)
-    '''
-    conn = sqlite3.connect(core_inst.queueDB, timeout=30)
-    c = conn.cursor()
-
-    try:
-        c.execute('DELETE FROM commands;')
-        conn.commit()
-    except:
-        pass
-
-    conn.close()
-    events.event('queue_clear', onionr = core_inst.onionrInst)

onionr/coredb/keydb/__init__.py

@@ -1 +0,0 @@
-from . import addkeys, listkeys, removekeys, userinfo, transportinfo

onionr/coredb/keydb/addkeys.py

@@ -1,91 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    add user keys or transport addresses
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import sqlite3
-import onionrevents as events
-from onionrutils import stringvalidators
-
-def add_peer(core_inst, peerID, name=''):
-    '''
-        Adds a public key to the key database (misleading function name)
-    '''
-    if peerID in core_inst.listPeers() or peerID == core_inst._crypto.pubKey:
-        raise ValueError("specified id is already known")
-
-    # This function simply adds a peer to the DB
-    if not stringvalidators.validate_pub_key(peerID):
-        return False
-
-    events.event('pubkey_add', data = {'key': peerID}, onionr = core_inst.onionrInst)
-
-    conn = sqlite3.connect(core_inst.peerDB, timeout=30)
-    hashID = core_inst._crypto.pubKeyHashID(peerID)
-    c = conn.cursor()
-    t = (peerID, name, 'unknown', hashID, 0)
-
-    for i in c.execute("SELECT * FROM peers WHERE id = ?;", (peerID,)):
-        try:
-            if i[0] == peerID:
-                conn.close()
-                return False
-        except ValueError:
-            pass
-        except IndexError:
-            pass
-    c.execute('INSERT INTO peers (id, name, dateSeen, hashID, trust) VALUES(?, ?, ?, ?, ?);', t)
-    conn.commit()
-    conn.close()
-
-    return True
-
-def add_address(core_inst, address):
-    '''
-        Add an address to the address database (only tor currently)
-    '''
-
-    if type(address) is None or len(address) == 0:
-        return False
-    if stringvalidators.validate_transport(address):
-        if address == core_inst.config.get('i2p.ownAddr', None) or address == core_inst.hsAddress:
-            return False
-        conn = sqlite3.connect(core_inst.addressDB, timeout=30)
-        c = conn.cursor()
-        # check if address is in database
-        # this is safe to do because the address is validated above, but we strip some chars here too just in case
-        address = address.replace('\'', '').replace(';', '').replace('"', '').replace('\\', '')
-        for i in c.execute("SELECT * FROM adders WHERE address = ?;", (address,)):
-            try:
-                if i[0] == address:
-                    conn.close()
-                    return False
-            except ValueError:
-                pass
-            except IndexError:
-                pass
-
-        t = (address, 1)
-        c.execute('INSERT INTO adders (address, type) VALUES(?, ?);', t)
-        conn.commit()
-        conn.close()
-
-        events.event('address_add', data = {'address': address}, onionr = core_inst.onionrInst)
-
-        return True
-    else:
-        return False

onionr/coredb/keydb/listkeys.py

@@ -1,83 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    get lists for user keys or transport addresses
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import sqlite3
-import logger
-from onionrutils import epoch
-def list_peers(core_inst, randomOrder=True, getPow=False, trust=0):
-    '''
-        Return a list of public keys (misleading function name)
-
-        randomOrder determines if the list should be in a random order
-        trust sets the minimum trust to list
-    '''
-    conn = sqlite3.connect(core_inst.peerDB, timeout=30)
-    c = conn.cursor()
-
-    payload = ''
-
-    if trust not in (0, 1, 2):
-        logger.error('Tried to select invalid trust.')
-        return
-
-    if randomOrder:
-        payload = 'SELECT * FROM peers WHERE trust >= ? ORDER BY RANDOM();'
-    else:
-        payload = 'SELECT * FROM peers WHERE trust >= ?;'
-
-    peerList = []
-
-    for i in c.execute(payload, (trust,)):
-        try:
-            if len(i[0]) != 0:
-                if getPow:
-                    peerList.append(i[0] + '-' + i[1])
-                else:
-                    peerList.append(i[0])
-        except TypeError:
-            pass
-
-    conn.close()
-
-    return peerList
-
-def list_adders(core_inst, randomOrder=True, i2p=True, recent=0):
-    '''
-        Return a list of transport addresses
-    '''
-    conn = sqlite3.connect(core_inst.addressDB, timeout=30)
-    c = conn.cursor()
-    if randomOrder:
-        addresses = c.execute('SELECT * FROM adders ORDER BY RANDOM();')
-    else:
-        addresses = c.execute('SELECT * FROM adders;')
-    addressList = []
-    for i in addresses:
-        if len(i[0].strip()) == 0:
-            continue
-        addressList.append(i[0])
-    conn.close()
-    testList = list(addressList) # create new list to iterate
-    for address in testList:
-        try:
-            if recent > 0 and (epoch.get_epoch() - core_inst.getAddressInfo(address, 'lastConnect')) > recent:
-                raise TypeError # If there is no last-connected date or it was too long ago, don't add peer to list if recent is not 0
-        except TypeError:
-            addressList.remove(address)
-    return addressList

onionr/coredb/keydb/removekeys.py

@@ -1,40 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    Remove a transport address but don't ban them
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import sqlite3
-import onionrevents as events
-from onionrutils import stringvalidators
-
-def remove_address(core_inst, address):
-    '''
-        Remove an address from the address database
-    '''
-
-    if stringvalidators.validate_transport(address):
-        conn = sqlite3.connect(core_inst.addressDB, timeout=30)
-        c = conn.cursor()
-        t = (address,)
-        c.execute('Delete from adders where address=?;', t)
-        conn.commit()
-        conn.close()
-
-        events.event('address_remove', data = {'address': address}, onionr = core_inst.onionrInst)
-        return True
-    else:
-        return False

onionr/coredb/keydb/transportinfo.py

@@ -1,72 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    get or set transport address meta information
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import sqlite3
-def get_address_info(core_inst, address, info):
-    '''
-        Get info about an address from its database entry
-
-        address text, 0
-        type int, 1
-        knownPeer text, 2
-        speed int, 3
-        success int, 4
-        powValue    5
-        failure int 6
-        lastConnect 7
-        trust       8
-        introduced  9
-    '''
-
-    conn = sqlite3.connect(core_inst.addressDB, timeout=30)
-    c = conn.cursor()
-
-    command = (address,)
-    infoNumbers = {'address': 0, 'type': 1, 'knownPeer': 2, 'speed': 3, 'success': 4, 'powValue': 5, 'failure': 6, 'lastConnect': 7, 'trust': 8, 'introduced': 9}
-    info = infoNumbers[info]
-    iterCount = 0
-    retVal = ''
-
-    for row in c.execute('SELECT * FROM adders WHERE address=?;', command):
-        for i in row:
-            if iterCount == info:
-                retVal = i
-                break
-            else:
-                iterCount += 1
-    conn.close()
-
-    return retVal
-
-def set_address_info(core_inst, address, key, data):
-    '''
-        Update an address for a key
-    '''
-
-    conn = sqlite3.connect(core_inst.addressDB, timeout=30)
-    c = conn.cursor()
-
-    command = (data, address)
-
-    if key not in ('address', 'type', 'knownPeer', 'speed', 'success', 'failure', 'powValue', 'lastConnect', 'lastConnectAttempt', 'trust', 'introduced'):
-        raise Exception("Got invalid database key when setting address info")
-    else:
-        c.execute('UPDATE adders SET ' + key + ' = ? WHERE address=?', command)
-        conn.commit()
-    conn.close()

onionr/coredb/keydb/userinfo.py

@@ -1,69 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    get or set information about a user id
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import sqlite3
-def get_user_info(core_inst, peer, info):
-    '''
-        Get info about a peer from their database entry
-
-        id text             0
-        name text,          1
-        adders text,        2
-        dateSeen not null,  3
-        trust int           4
-        hashID text         5
-    '''
-    conn = sqlite3.connect(core_inst.peerDB, timeout=30)
-    c = conn.cursor()
-
-    command = (peer,)
-    infoNumbers = {'id': 0, 'name': 1, 'adders': 2, 'dateSeen': 3, 'trust': 4, 'hashID': 5}
-    info = infoNumbers[info]
-    iterCount = 0
-    retVal = ''
-
-    for row in c.execute('SELECT * FROM peers WHERE id=?;', command):
-        for i in row:
-            if iterCount == info:
-                retVal = i
-                break
-            else:
-                iterCount += 1
-
-    conn.close()
-
-    return retVal
-
-def set_peer_info(core_inst, peer, key, data):
-    '''
-        Update a peer for a key
-    '''
-
-    conn = sqlite3.connect(core_inst.peerDB, timeout=30)
-    c = conn.cursor()
-
-    command = (data, peer)
-
-    # TODO: validate key on whitelist
-    if key not in ('id', 'name', 'pubkey', 'forwardKey', 'dateSeen', 'trust'):
-        raise Exception("Got invalid database key when setting peer info")
-
-    c.execute('UPDATE peers SET ' + key + ' = ? WHERE id=?', command)
-    conn.commit()
-    conn.close()

onionr/dbcreator.py

@@ -1,156 +0,0 @@
-'''
-    Onionr - Private P2P Communication
-
-    DBCreator, creates sqlite3 databases used by Onionr
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import sqlite3, os
-class DBCreator:
-    def __init__(self, coreInst):
-        self.core = coreInst
-
-    def createAddressDB(self):
-        '''
-            Generate the address database
-
-            types:
-                1: I2P b32 address
-                2: Tor v2 (like facebookcorewwwi.onion)
-                3: Tor v3
-        '''
-        conn = sqlite3.connect(self.core.addressDB)
-        c = conn.cursor()
-        c.execute('''CREATE TABLE adders(
-            address text,
-            type int,
-            knownPeer text,
-            speed int,
-            success int,
-            powValue text,
-            failure int,
-            lastConnect int,
-            lastConnectAttempt int,
-            trust int,
-            introduced int
-            );
-        ''')
-        conn.commit()
-        conn.close()
-
-    def createPeerDB(self):
-        '''
-            Generate the peer sqlite3 database and populate it with the peers table.
-        '''
-        # generate the peer database
-        conn = sqlite3.connect(self.core.peerDB)
-        c = conn.cursor()
-        c.execute('''CREATE TABLE peers(
-            ID text not null,
-            name text,
-            adders text,
-            dateSeen not null,
-            trust int,
-            hashID text);
-        ''')
-        c.execute('''CREATE TABLE forwardKeys(
-        peerKey text not null,
-        forwardKey text not null,
-        date int not null,
-        expire int not null
-        );''')
-        conn.commit()
-        conn.close()
-        return
-
-    def createBlockDB(self):
-        '''
-            Create a database for blocks
-
-            hash         - the hash of a block
-            dateReceived - the date the block was recieved, not necessarily when it was created
-            decrypted    - if we can successfully decrypt the block (does not describe its current state)
-            dataType     - data type of the block
-            dataFound    - if the data has been found for the block
-            dataSaved    - if the data has been saved for the block
-            sig    - optional signature by the author (not optional if author is specified)
-            author       - multi-round partial sha3-256 hash of authors public key
-            dateClaimed  - timestamp claimed inside the block, only as trustworthy as the block author is
-            expire int   - block expire date in epoch
-        '''
-        if os.path.exists(self.core.blockDB):
-            raise FileExistsError("Block database already exists")
-        conn = sqlite3.connect(self.core.blockDB)
-        c = conn.cursor()
-        c.execute('''CREATE TABLE hashes(
-            hash text not null,
-            dateReceived int,
-            decrypted int,
-            dataType text,
-            dataFound int,
-            dataSaved int,
-            sig text,
-            author text,
-            dateClaimed int,
-            expire int
-            );
-        ''')
-        conn.commit()
-        conn.close()
-        return
-    
-    def createBlockDataDB(self):
-        if os.path.exists(self.core.blockDataDB):
-            raise FileExistsError("Block data database already exists")
-        conn = sqlite3.connect(self.core.blockDataDB)
-        c = conn.cursor()
-        c.execute('''CREATE TABLE blockData(
-            hash text not null,
-            data blob not null
-            );
-        ''')
-        conn.commit()
-        conn.close()
-
-    def createForwardKeyDB(self):
-        '''
-            Create the forward secrecy key db (*for *OUR* keys*)
-        '''
-        if os.path.exists(self.core.forwardKeysFile):
-            raise FileExistsError("Block database already exists")
-        conn = sqlite3.connect(self.core.forwardKeysFile)
-        c = conn.cursor()
-        c.execute('''CREATE TABLE myForwardKeys(
-            peer text not null,
-            publickey text not null,
-            privatekey text not null,
-            date int not null,
-            expire int not null
-            );
-        ''')
-        conn.commit()
-        conn.close()
-        return
-    
-    def createDaemonDB(self):
-        '''
-            Create the daemon queue database
-        '''
-        conn = sqlite3.connect(self.core.queueDB, timeout=10)
-        c = conn.cursor()
-        # Create table
-        c.execute('''CREATE TABLE commands (id integer primary key autoincrement, command text, data text, date text, responseID text)''')
-        conn.commit()
-        conn.close()

onionr/etc/README.md

@@ -1,11 +0,0 @@
-# etc
-
-Files that don't really fit anywhere else, but aren't used very frequently.
-
-## Files
-
-humanreadabletime.py: take integer seconds and return a human readable time string
-
-pgpwords.py: represent data using the pgp word list: https://en.wikipedia.org/wiki/PGP_word_list
-
-onionrvalues.py: spec values for onionr blocks and other things