fix forms and refactor node ingest/validation
This commit is contained in:
parent
dc06c87808
commit
137a95217d
2
bin/cmd
2
bin/cmd
|
@ -1,7 +1,7 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
source .venv/bin/activate
|
source .venv/bin/activate
|
||||||
export FLASK_APP=nodes/app.py
|
export FLASK_APP=xmrnodes/app.py
|
||||||
export FLASK_SECRETS=config.py
|
export FLASK_SECRETS=config.py
|
||||||
export FLASK_DEBUG=1
|
export FLASK_DEBUG=1
|
||||||
flask $1
|
flask $1
|
||||||
|
|
123
xmrnodes/app.py
123
xmrnodes/app.py
|
@ -1,19 +1,28 @@
|
||||||
import json
|
import json
|
||||||
import requests
|
import requests
|
||||||
import re
|
import re
|
||||||
|
import logging
|
||||||
from os import makedirs
|
from os import makedirs
|
||||||
|
from datetime import datetime
|
||||||
from flask import Flask, request, redirect
|
from flask import Flask, request, redirect
|
||||||
from flask import render_template, flash, url_for
|
from flask import render_template, flash, url_for
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
from xmrnodes.forms import SubmitNode
|
||||||
from xmrnodes.models import Node
|
from xmrnodes.models import Node
|
||||||
|
|
||||||
|
|
||||||
|
logging.basicConfig(
|
||||||
|
level=logging.INFO,
|
||||||
|
format='%(asctime)s - %(levelname)s - %(message)s'
|
||||||
|
)
|
||||||
|
|
||||||
app = Flask(__name__)
|
app = Flask(__name__)
|
||||||
app.config.from_envvar("FLASK_SECRETS")
|
app.config.from_envvar("FLASK_SECRETS")
|
||||||
app.secret_key = app.config["SECRET_KEY"]
|
app.secret_key = app.config["SECRET_KEY"]
|
||||||
|
|
||||||
@app.route("/", methods=["GET", "POST"])
|
@app.route("/", methods=["GET", "POST"])
|
||||||
def index():
|
def index():
|
||||||
|
form = SubmitNode()
|
||||||
itp = 20
|
itp = 20
|
||||||
page = request.args.get("page", 1)
|
page = request.args.get("page", 1)
|
||||||
try:
|
try:
|
||||||
|
@ -22,61 +31,85 @@ def index():
|
||||||
flash("Wow, wtf hackerman. Cool it.")
|
flash("Wow, wtf hackerman. Cool it.")
|
||||||
page = 1
|
page = 1
|
||||||
|
|
||||||
nodes = Node.select().where(Node.available==True).order_by(Node.datetime_entered.desc()).paginate(page, itp)
|
nodes = Node.select().where(Node.available==True).order_by(
|
||||||
|
Node.datetime_entered.desc()
|
||||||
|
).paginate(page, itp)
|
||||||
total_pages = Node.select().count() / itp
|
total_pages = Node.select().count() / itp
|
||||||
return render_template("index.html", nodes=nodes, page=page, total_pages=total_pages)
|
return render_template(
|
||||||
|
"index.html",
|
||||||
|
nodes=nodes,
|
||||||
|
page=page,
|
||||||
|
total_pages=total_pages,
|
||||||
|
form=form
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@app.route("/add", methods=["GET", "POST"])
|
@app.route("/add", methods=["GET", "POST"])
|
||||||
def add():
|
def add():
|
||||||
if request.method == "POST":
|
if request.method == "POST":
|
||||||
url = request.form.get("url")
|
url = request.form.get("node_url")
|
||||||
regex = re.compile(
|
regex = re.compile(
|
||||||
r'^(?:http)s?://' # http:// or https://
|
r'^(?:http)s?://' # http:// or https://
|
||||||
r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain...
|
r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain...
|
||||||
r'localhost|' #localhost...
|
r'localhost|' #localhost...
|
||||||
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
|
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
|
||||||
r'(?::\d+)?' # optional port
|
r'(?::\d+)?' # optional port
|
||||||
r'(?:/?|[/?]\S+)$', re.IGNORECASE)
|
r'(?:/?|[/?]\S+)$', re.IGNORECASE
|
||||||
|
|
||||||
re_match = re.match(regex, url)
|
|
||||||
|
|
||||||
if re_match is not None:
|
|
||||||
_url = urlparse(url)
|
|
||||||
try:
|
|
||||||
endpoint = f"{_url.scheme}://{_url.netloc}"
|
|
||||||
r = requests.get(endpoint + "/get_info", timeout=3)
|
|
||||||
r.raise_for_status()
|
|
||||||
# print(r.json())
|
|
||||||
return {"status": "success"}
|
|
||||||
except requests.exceptions.ConnectTimeout:
|
|
||||||
flash("connection timed out. double-check the port")
|
|
||||||
return {"status": "fail", "reason": "timeout"}
|
|
||||||
except requests.exceptions.SSLError:
|
|
||||||
flash("invalid certificate")
|
|
||||||
return {"status": "fail", "reason": "invalid cert"}
|
|
||||||
except Exception as e:
|
|
||||||
flash("failed to send req", str(e))
|
|
||||||
print(e)
|
|
||||||
return {"status": "fail"}
|
|
||||||
else:
|
|
||||||
flash("invalid url provided")
|
|
||||||
return {"status": "fail"}
|
|
||||||
|
|
||||||
return "ok"
|
|
||||||
node = Node(
|
|
||||||
scheme=proto,
|
|
||||||
address=addr,
|
|
||||||
port=port,
|
|
||||||
version=r.json()["version"],
|
|
||||||
tor=addr.endswith(".onion"),
|
|
||||||
available=r.json()["status"] == "OK",
|
|
||||||
mainnet=r.json()["mainnet"],
|
|
||||||
)
|
)
|
||||||
node.save()
|
re_match = re.match(regex, url)
|
||||||
return {"status": "success"}
|
if re_match is None:
|
||||||
|
flash("This doesn't look like a valid URL")
|
||||||
|
else:
|
||||||
|
_url = urlparse(url)
|
||||||
|
url = f"{_url.scheme}://{_url.netloc}"
|
||||||
|
if Node.select().where(Node.url == url).exists():
|
||||||
|
flash("This node is already in the database.")
|
||||||
|
else:
|
||||||
|
flash("Seems like a valid node. Added to the database and will check soon.")
|
||||||
|
node = Node(url=url)
|
||||||
|
node.save()
|
||||||
return redirect("/")
|
return redirect("/")
|
||||||
|
|
||||||
|
@app.cli.command("validate")
|
||||||
|
def validate():
|
||||||
|
nodes = Node.select().where(Node.validated == False)
|
||||||
|
for node in nodes:
|
||||||
|
now = datetime.now()
|
||||||
|
is_onion = node.url.split(":")[1].endswith(".onion")
|
||||||
|
logging.info(f"Attempting to validate {node.url}")
|
||||||
|
if is_onion:
|
||||||
|
logging.info("onion address found")
|
||||||
|
node.tor = True
|
||||||
|
try:
|
||||||
|
r = requests.get(node.url + "/get_info", timeout=3)
|
||||||
|
r.raise_for_status()
|
||||||
|
assert "height" in r.json()
|
||||||
|
assert "nettype" in r.json()
|
||||||
|
nettype = r.json()["nettype"]
|
||||||
|
logging.info("success")
|
||||||
|
if nettype in ["mainnet", "stagenet", "testnet"]:
|
||||||
|
node.nettype = nettype
|
||||||
|
node.available = True
|
||||||
|
node.validated = True
|
||||||
|
node.datetime_checked = now
|
||||||
|
node.save()
|
||||||
|
else:
|
||||||
|
logging.info("unexpected nettype")
|
||||||
|
except requests.exceptions.ConnectTimeout:
|
||||||
|
logging.info("connection timed out")
|
||||||
|
node.delete_instance()
|
||||||
|
except requests.exceptions.SSLError:
|
||||||
|
logging.info("invalid certificate")
|
||||||
|
node.delete_instance()
|
||||||
|
except requests.exceptions.ConnectionError:
|
||||||
|
logging.info("connection error")
|
||||||
|
node.delete_instance()
|
||||||
|
except requests.exceptions.HTTPError:
|
||||||
|
logging.info("http error, 4xx or 5xx")
|
||||||
|
node.delete_instance()
|
||||||
|
except Exception as e:
|
||||||
|
logging.info("failed for reasons unknown")
|
||||||
|
node.delete_instance()
|
||||||
|
|
||||||
@app.route("/about")
|
@app.route("/about")
|
||||||
def about():
|
def about():
|
||||||
|
|
|
@ -0,0 +1,7 @@
|
||||||
|
from flask_wtf import FlaskForm
|
||||||
|
from wtforms import StringField
|
||||||
|
from wtforms.validators import DataRequired
|
||||||
|
|
||||||
|
|
||||||
|
class SubmitNode(FlaskForm):
|
||||||
|
node_url = StringField('Node URL:', validators=[DataRequired()])
|
|
@ -4,19 +4,17 @@ from xmrnodes import config
|
||||||
|
|
||||||
|
|
||||||
data_dir = getattr(config, 'DATA_FOLDER', './data')
|
data_dir = getattr(config, 'DATA_FOLDER', './data')
|
||||||
db = SqliteDatabase(f"{data_dir}/db/sqlite.db")
|
db = SqliteDatabase(f"{data_dir}/sqlite.db")
|
||||||
|
|
||||||
class Node(Model):
|
class Node(Model):
|
||||||
id = AutoField()
|
id = AutoField()
|
||||||
scheme = CharField()
|
url = CharField()
|
||||||
address = CharField()
|
|
||||||
port = IntegerField()
|
|
||||||
version = CharField(null=True)
|
|
||||||
tor = BooleanField(default=False)
|
tor = BooleanField(default=False)
|
||||||
available = BooleanField(default=False)
|
available = BooleanField(default=False)
|
||||||
mainnet = BooleanField(default=False)
|
validated = BooleanField(default=False)
|
||||||
|
nettype = CharField(null=True)
|
||||||
datetime_entered = DateTimeField(default=datetime.now)
|
datetime_entered = DateTimeField(default=datetime.now)
|
||||||
datetime_checked = DateTimeField(default=datetime.now)
|
datetime_checked = DateTimeField(default=None, null=True)
|
||||||
datetime_failed = DateTimeField(default=None, null=True)
|
datetime_failed = DateTimeField(default=None, null=True)
|
||||||
|
|
||||||
class Meta:
|
class Meta:
|
||||||
|
|
|
@ -9,13 +9,25 @@
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{% for node in nodes %}
|
{% for node in nodes %}
|
||||||
{{ node }}<br>
|
{{ node.url }}<br>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
|
|
||||||
<form id="addnode" method="POST">
|
<form method="POST" action="{{ url_for('add') }}">
|
||||||
<label>Node URL:</label>
|
{{ form.csrf_token }}
|
||||||
<input type="text" name="url"/>
|
{% for f in form %}
|
||||||
<input type="submit" value="Submit" name="submit" class="submit" id="submit" />
|
{% if f.name != 'csrf_token' %}
|
||||||
|
<div class="form-group">
|
||||||
|
{{ f.label }}
|
||||||
|
{{ f }}
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
{% endfor %}
|
||||||
|
<ul>
|
||||||
|
{% for field, errors in form.errors.items() %}
|
||||||
|
<li>{{ form[field].label }}: {{ ', '.join(errors) }}</li>
|
||||||
|
{% endfor %}
|
||||||
|
</ul>
|
||||||
|
<input type="submit" value="Send" class="btn btn-link btn-outline btn-xl">
|
||||||
</form>
|
</form>
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue