kev
/
Onionr
1
0
Fork 0
Code Issues 24 Pull Requests Projects Releases 2 Wiki Activity
Onionr/docs/html/onionr/onionrproofs/subprocesspow.html

339 lines
17 KiB
HTML
Raw Normal View History

added auto generated api documentation
2019-09-23 23:20:14 +00:00
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, minimum-scale=1" />
<meta name="generator" content="pdoc 0.6.3" />
<title>onionr.onionrproofs.subprocesspow API documentation</title>
<meta name="description" content="Onionr - Private P2P Communication …" />
<link href='https://cdnjs.cloudflare.com/ajax/libs/normalize/8.0.0/normalize.min.css' rel='stylesheet'>
<link href='https://cdnjs.cloudflare.com/ajax/libs/10up-sanitize.css/8.0.0/sanitize.min.css' rel='stylesheet'>
<link href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css" rel="stylesheet">
<style>.flex{display:flex !important}body{line-height:1.5em}#content{padding:20px}#sidebar{padding:30px;overflow:hidden}.http-server-breadcrumbs{font-size:130%;margin:0 0 15px 0}#footer{font-size:.75em;padding:5px 30px;border-top:1px solid #ddd;text-align:right}#footer p{margin:0 0 0 1em;display:inline-block}#footer p:last-child{margin-right:30px}h1,h2,h3,h4,h5{font-weight:300}h1{font-size:2.5em;line-height:1.1em}h2{font-size:1.75em;margin:1em 0 .50em 0}h3{font-size:1.4em;margin:25px 0 10px 0}h4{margin:0;font-size:105%}a{color:#058;text-decoration:none;transition:color .3s ease-in-out}a:hover{color:#e82}.title code{font-weight:bold}h2[id^="header-"]{margin-top:2em}.ident{color:#900}pre code{background:#f8f8f8;font-size:.8em;line-height:1.4em}code{background:#f2f2f1;padding:1px 4px;overflow-wrap:break-word}h1 code{background:transparent}pre{background:#f8f8f8;border:0;border-top:1px solid #ccc;border-bottom:1px solid #ccc;margin:1em 0;padding:1ex}#http-server-module-list{display:flex;flex-flow:column}#http-server-module-list div{display:flex}#http-server-module-list dt{min-width:10%}#http-server-module-list p{margin-top:0}.toc ul,#index{list-style-type:none;margin:0;padding:0}#index code{background:transparent}#index h3{border-bottom:1px solid #ddd}#index ul{padding:0}#index h4{font-weight:bold}#index h4 + ul{margin-bottom:.6em}@media (min-width:200ex){#index .two-column{column-count:2}}@media (min-width:300ex){#index .two-column{column-count:3}}dl{margin-bottom:2em}dl dl:last-child{margin-bottom:4em}dd{margin:0 0 1em 3em}#header-classes + dl > dd{margin-bottom:3em}dd dd{margin-left:2em}dd p{margin:10px 0}.name{background:#eee;font-weight:bold;font-size:.85em;padding:5px 10px;display:inline-block;min-width:40%}.name:hover{background:#e0e0e0}.name > span:first-child{white-space:nowrap}.name.class > span:nth-child(2){margin-left:.4em}.inherited{color:#999;border-left:5px solid #eee;padding-left:1em}.inheritance em{font-style:normal;font-weight:bold}.desc h2{font-weight:400;font-size:1.25em}.desc h3{font-size:1em}.desc dt code{background:inherit}.source summary{color:#666;text-align:right;font-weight:400;font-size:.8em;text-transform:uppercase;cursor:pointer}.source pre{max-height:500px;overflow:auto;margin:0}.source pre code{font-size:12px;overflow:visible}.hlist{list-style:none}.hlist li{display:inline}.hlist li:after{content:',\2002'}.hlist li:last-child:after{content:none}.hlist .hlist{display:inline;padding-left:1em}img{max-width:100%}.admonition{padding:.1em .5em;margin-bottom:1em}.admonition-title{font-weight:bold}.admonition.note,.admonition.info,.admonition.important{background:#aef}.admonition.todo,.admonition.versionadded,.admonition.tip,.admonition.hint{background:#dfd}.admonition.warning,.admonition.versionchanged,.admonition.deprecated{background:#fd4}.admonition.error,.admonition.danger,.admonition.caution{background:lightpink}</style>
<style media="screen and (min-width: 700px)">@media screen and (min-width:700px){#sidebar{width:30%}#content{width:70%;max-width:100ch;padding:3em 4em;border-left:1px solid #ddd}pre code{font-size:1em}.item .name{font-size:1em}main{display:flex;flex-direction:row-reverse;justify-content:flex-end}.toc ul ul,#index ul{padding-left:1.5em}.toc > ul > li{margin-top:.5em}}</style>
<style media="print">@media print{#sidebar h1{page-break-before:always}.source{display:none}}@media print{*{background:transparent !important;color:#000 !important;box-shadow:none !important;text-shadow:none !important}a[href]:after{content:" (" attr(href) ")";font-size:90%}a[href][title]:after{content:none}abbr[title]:after{content:" (" attr(title) ")"}.ir a:after,a[href^="javascript:"]:after,a[href^="#"]:after{content:""}pre,blockquote{border:1px solid #999;page-break-inside:avoid}thead{display:table-header-group}tr,img{page-break-inside:avoid}img{max-width:100% !important}@page{margin:0.5cm}p,h2,h3{orphans:3;widows:3}h1,h2,h3,h4,h5,h6{page-break-after:avoid}}</style>
</head>
<body>
<main>
<article id="content">
<header>
<h1 class="title">Module <code>onionr.onionrproofs.subprocesspow</code></h1>
</header>
<section id="section-intro">
<p>Onionr - Private P2P Communication</p>
<p>Multiprocess proof of work</p>
<details class="source">
<summary>Source code</summary>
<pre><code class="python">#!/usr/bin/env python3
&#39;&#39;&#39;
Onionr - Private P2P Communication
Multiprocess proof of work
&#39;&#39;&#39;
&#39;&#39;&#39;
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see &lt;https://www.gnu.org/licenses/&gt;.
&#39;&#39;&#39;
import subprocess, os
import multiprocessing, threading, time, json
from multiprocessing import Pipe, Process
from onionrblocks import onionrblockapi
import config, onionrutils, logger, onionrproofs, onionrcrypto as crypto
from onionrutils import bytesconverter
class SubprocessPOW:
def __init__(self, data, metadata, subproc_count=None):
&#39;&#39;&#39;
Onionr proof of work using multiple processes
Accepts block data, block metadata
if subproc_count is not set, os.cpu_count() is used to determine the number of processes
Do to Python GIL multiprocessing or use of external libraries is necessary to accelerate CPU bound tasks
&#39;&#39;&#39;
# No known benefit to using more processes than there are cores.
# Note: os.cpu_count perhaps not always accurate
if subproc_count is None:
subproc_count = os.cpu_count()
self.subproc_count = subproc_count
self.result = &#39;&#39;
self.shutdown = False
self.data = data
self.metadata = metadata
# dump dict to measure bytes of json metadata. Cannot reuse later because the pow token must be added
json_metadata = json.dumps(metadata).encode()
self.data = bytesconverter.str_to_bytes(data)
# Calculate difficulty. Dumb for now, may use good algorithm in the future.
self.difficulty = onionrproofs.getDifficultyForNewBlock(bytes(json_metadata + b&#39;\n&#39; + self.data))
logger.info(&#39;Computing POW (difficulty: %s)...&#39; % (self.difficulty,))
self.mainHash = &#39;0&#39; * 64
self.puzzle = self.mainHash[0:min(self.difficulty, len(self.mainHash))]
self.shutdown = False
self.payload = None
def start(self):
# Create a new thread for each subprocess
for x in range(self.subproc_count):
threading.Thread(target=self._spawn_proc).start()
# Monitor the processes for a payload, shut them down when its found
while True:
if self.payload is None:
time.sleep(0.1)
else:
self.shutdown = True
return self.payload
def _spawn_proc(self):
# Create a child proof of work process, wait for data and send shutdown signal when its found
parent_conn, child_conn = Pipe()
p = Process(target=self.do_pow, args=(child_conn,))
p.start()
p.join()
payload = None
try:
while True:
data = parent_conn.recv()
if len(data) &gt;= 1:
payload = data
break
except KeyboardInterrupt:
pass
finally:
parent_conn.send(&#39;shutdown&#39;)
self.payload = payload
def do_pow(self, pipe):
nonce = -10000000 # Start nonce at negative 10 million so that the chosen nonce is likely to be small in length
nonceStart = nonce
data = self.data
metadata = self.metadata
puzzle = self.puzzle
difficulty = self.difficulty
while True:
# Break if shutdown received
if pipe.poll() and pipe.recv() == &#39;shutdown&#39;:
break
# Load nonce into block metadata
metadata[&#39;pow&#39;] = nonce
# Serialize metadata, combine with block data
payload = json.dumps(metadata).encode() + b&#39;\n&#39; + data
# Check sha3_256 hash of block, compare to puzzle. Send payload if puzzle finished
token = crypto.hashers.sha3_hash(payload)
token = bytesconverter.bytes_to_str(token) # ensure token is string
if puzzle == token[0:difficulty]:
pipe.send(payload)
break
nonce += 1
</code></pre>
</details>
</section>
<section>
</section>
<section>
</section>
<section>
</section>
<section>
<h2 class="section-title" id="header-classes">Classes</h2>
<dl>
<dt id="onionr.onionrproofs.subprocesspow.SubprocessPOW"><code class="flex name class">
<span>class <span class="ident">SubprocessPOW</span></span>
<span>(</span><span>data, metadata, subproc_count=None)</span>
</code></dt>
<dd>
<section class="desc"><p>Onionr proof of work using multiple processes
Accepts block data, block metadata
if subproc_count is not set, os.cpu_count() is used to determine the number of processes</p>
<p>Do to Python GIL multiprocessing or use of external libraries is necessary to accelerate CPU bound tasks</p></section>
<details class="source">
<summary>Source code</summary>
<pre><code class="python">class SubprocessPOW:
def __init__(self, data, metadata, subproc_count=None):
&#39;&#39;&#39;
Onionr proof of work using multiple processes
Accepts block data, block metadata
if subproc_count is not set, os.cpu_count() is used to determine the number of processes
Do to Python GIL multiprocessing or use of external libraries is necessary to accelerate CPU bound tasks
&#39;&#39;&#39;
# No known benefit to using more processes than there are cores.
# Note: os.cpu_count perhaps not always accurate
if subproc_count is None:
subproc_count = os.cpu_count()
self.subproc_count = subproc_count
self.result = &#39;&#39;
self.shutdown = False
self.data = data
self.metadata = metadata
# dump dict to measure bytes of json metadata. Cannot reuse later because the pow token must be added
json_metadata = json.dumps(metadata).encode()
self.data = bytesconverter.str_to_bytes(data)
# Calculate difficulty. Dumb for now, may use good algorithm in the future.
self.difficulty = onionrproofs.getDifficultyForNewBlock(bytes(json_metadata + b&#39;\n&#39; + self.data))
logger.info(&#39;Computing POW (difficulty: %s)...&#39; % (self.difficulty,))
self.mainHash = &#39;0&#39; * 64
self.puzzle = self.mainHash[0:min(self.difficulty, len(self.mainHash))]
self.shutdown = False
self.payload = None
def start(self):
# Create a new thread for each subprocess
for x in range(self.subproc_count):
threading.Thread(target=self._spawn_proc).start()
# Monitor the processes for a payload, shut them down when its found
while True:
if self.payload is None:
time.sleep(0.1)
else:
self.shutdown = True
return self.payload
def _spawn_proc(self):
# Create a child proof of work process, wait for data and send shutdown signal when its found
parent_conn, child_conn = Pipe()
p = Process(target=self.do_pow, args=(child_conn,))
p.start()
p.join()
payload = None
try:
while True:
data = parent_conn.recv()
if len(data) &gt;= 1:
payload = data
break
except KeyboardInterrupt:
pass
finally:
parent_conn.send(&#39;shutdown&#39;)
self.payload = payload
def do_pow(self, pipe):
nonce = -10000000 # Start nonce at negative 10 million so that the chosen nonce is likely to be small in length
nonceStart = nonce
data = self.data
metadata = self.metadata
puzzle = self.puzzle
difficulty = self.difficulty
while True:
# Break if shutdown received
if pipe.poll() and pipe.recv() == &#39;shutdown&#39;:
break
# Load nonce into block metadata
metadata[&#39;pow&#39;] = nonce
# Serialize metadata, combine with block data
payload = json.dumps(metadata).encode() + b&#39;\n&#39; + data
# Check sha3_256 hash of block, compare to puzzle. Send payload if puzzle finished
token = crypto.hashers.sha3_hash(payload)
token = bytesconverter.bytes_to_str(token) # ensure token is string
if puzzle == token[0:difficulty]:
pipe.send(payload)
break
nonce += 1</code></pre>
</details>
<h3>Methods</h3>
<dl>
<dt id="onionr.onionrproofs.subprocesspow.SubprocessPOW.do_pow"><code class="name flex">
<span>def <span class="ident">do_pow</span></span>(<span>self, pipe)</span>
</code></dt>
<dd>
<section class="desc"></section>
<details class="source">
<summary>Source code</summary>
<pre><code class="python">def do_pow(self, pipe):
nonce = -10000000 # Start nonce at negative 10 million so that the chosen nonce is likely to be small in length
nonceStart = nonce
data = self.data
metadata = self.metadata
puzzle = self.puzzle
difficulty = self.difficulty
while True:
# Break if shutdown received
if pipe.poll() and pipe.recv() == &#39;shutdown&#39;:
break
# Load nonce into block metadata
metadata[&#39;pow&#39;] = nonce
# Serialize metadata, combine with block data
payload = json.dumps(metadata).encode() + b&#39;\n&#39; + data
# Check sha3_256 hash of block, compare to puzzle. Send payload if puzzle finished
token = crypto.hashers.sha3_hash(payload)
token = bytesconverter.bytes_to_str(token) # ensure token is string
if puzzle == token[0:difficulty]:
pipe.send(payload)
break
nonce += 1</code></pre>
</details>
</dd>
<dt id="onionr.onionrproofs.subprocesspow.SubprocessPOW.start"><code class="name flex">
<span>def <span class="ident">start</span></span>(<span>self)</span>
</code></dt>
<dd>
<section class="desc"></section>
<details class="source">
<summary>Source code</summary>
<pre><code class="python">def start(self):
# Create a new thread for each subprocess
for x in range(self.subproc_count):
threading.Thread(target=self._spawn_proc).start()
# Monitor the processes for a payload, shut them down when its found
while True:
if self.payload is None:
time.sleep(0.1)
else:
self.shutdown = True
return self.payload</code></pre>
</details>
</dd>
</dl>
</dd>
</dl>
</section>
</article>
<nav id="sidebar">
<h1>Index</h1>
<div class="toc">
<ul></ul>
</div>
<ul id="index">
<li><h3>Super-module</h3>
<ul>
<li><code><a title="onionr.onionrproofs" href="index.html">onionr.onionrproofs</a></code></li>
</ul>
</li>
<li><h3><a href="#header-classes">Classes</a></h3>
<ul>
<li>
<h4><code><a title="onionr.onionrproofs.subprocesspow.SubprocessPOW" href="#onionr.onionrproofs.subprocesspow.SubprocessPOW">SubprocessPOW</a></code></h4>
<ul class="">
<li><code><a title="onionr.onionrproofs.subprocesspow.SubprocessPOW.do_pow" href="#onionr.onionrproofs.subprocesspow.SubprocessPOW.do_pow">do_pow</a></code></li>
<li><code><a title="onionr.onionrproofs.subprocesspow.SubprocessPOW.start" href="#onionr.onionrproofs.subprocesspow.SubprocessPOW.start">start</a></code></li>
</ul>
</li>
</ul>
</li>
</ul>
</nav>
</main>
<footer id="footer">
<p>Generated by <a href="https://pdoc3.github.io/pdoc"><cite>pdoc</cite> 0.6.3</a>.</p>
</footer>
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js"></script>
<script>hljs.initHighlightingOnLoad()</script>
</body>
</html>