fix forms and refactor node ingest/validation
This commit is contained in:
lza_menace
parent
dc06c87808
commit
137a95217d
2
bin/cmd
2
bin/cmd
|
|
@ -1,7 +1,7 @@
|
|||
#!/bin/bash
|
||||
|
||||
source .venv/bin/activate
|
||||
export FLASK_APP=nodes/app.py
|
||||
export FLASK_APP=xmrnodes/app.py
|
||||
export FLASK_SECRETS=config.py
|
||||
export FLASK_DEBUG=1
|
||||
flask $1
|
||||
|
|
|
|||
123
xmrnodes/app.py
123
xmrnodes/app.py
|
|
@ -1,19 +1,28 @@
|
|||
import json
|
||||
import requests
|
||||
import re
|
||||
import logging
|
||||
from os import makedirs
|
||||
from datetime import datetime
|
||||
from flask import Flask, request, redirect
|
||||
from flask import render_template, flash, url_for
|
||||
from urllib.parse import urlparse
|
||||
from xmrnodes.forms import SubmitNode
|
||||
from xmrnodes.models import Node
|
||||
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
|
||||
app = Flask(__name__)
|
||||
app.config.from_envvar("FLASK_SECRETS")
|
||||
app.secret_key = app.config["SECRET_KEY"]
|
||||
|
||||
@app.route("/", methods=["GET", "POST"])
|
||||
def index():
|
||||
form = SubmitNode()
|
||||
itp = 20
|
||||
page = request.args.get("page", 1)
|
||||
try:
|
||||
|
|
@ -22,61 +31,85 @@ def index():
|
|||
flash("Wow, wtf hackerman. Cool it.")
|
||||
page = 1
|
||||
|
||||
nodes = Node.select().where(Node.available==True).order_by(Node.datetime_entered.desc()).paginate(page, itp)
|
||||
nodes = Node.select().where(Node.available==True).order_by(
|
||||
Node.datetime_entered.desc()
|
||||
).paginate(page, itp)
|
||||
total_pages = Node.select().count() / itp
|
||||
return render_template("index.html", nodes=nodes, page=page, total_pages=total_pages)
|
||||
return render_template(
|
||||
"index.html",
|
||||
nodes=nodes,
|
||||
page=page,
|
||||
total_pages=total_pages,
|
||||
form=form
|
||||
)
|
||||
|
||||
|
||||
@app.route("/add", methods=["GET", "POST"])
|
||||
def add():
|
||||
if request.method == "POST":
|
||||
url = request.form.get("url")
|
||||
url = request.form.get("node_url")
|
||||
regex = re.compile(
|
||||
r'^(?:http)s?://' # http:// or https://
|
||||
r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain...
|
||||
r'localhost|' #localhost...
|
||||
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
|
||||
r'(?::\d+)?' # optional port
|
||||
r'(?:/?|[/?]\S+)$', re.IGNORECASE)
|
||||
|
||||
re_match = re.match(regex, url)
|
||||
|
||||
if re_match is not None:
|
||||
_url = urlparse(url)
|
||||
try:
|
||||
endpoint = f"{_url.scheme}://{_url.netloc}"
|
||||
r = requests.get(endpoint + "/get_info", timeout=3)
|
||||
r.raise_for_status()
|
||||
# print(r.json())
|
||||
return {"status": "success"}
|
||||
except requests.exceptions.ConnectTimeout:
|
||||
flash("connection timed out. double-check the port")
|
||||
return {"status": "fail", "reason": "timeout"}
|
||||
except requests.exceptions.SSLError:
|
||||
flash("invalid certificate")
|
||||
return {"status": "fail", "reason": "invalid cert"}
|
||||
except Exception as e:
|
||||
flash("failed to send req", str(e))
|
||||
print(e)
|
||||
return {"status": "fail"}
|
||||
else:
|
||||
flash("invalid url provided")
|
||||
return {"status": "fail"}
|
||||
|
||||
return "ok"
|
||||
node = Node(
|
||||
scheme=proto,
|
||||
address=addr,
|
||||
port=port,
|
||||
version=r.json()["version"],
|
||||
tor=addr.endswith(".onion"),
|
||||
available=r.json()["status"] == "OK",
|
||||
mainnet=r.json()["mainnet"],
|
||||
r'^(?:http)s?://' # http:// or https://
|
||||
r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain...
|
||||
r'localhost|' #localhost...
|
||||
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
|
||||
r'(?::\d+)?' # optional port
|
||||
r'(?:/?|[/?]\S+)$', re.IGNORECASE
|
||||
)
|
||||
node.save()
|
||||
return {"status": "success"}
|
||||
re_match = re.match(regex, url)
|
||||
if re_match is None:
|
||||
flash("This doesn't look like a valid URL")
|
||||
else:
|
||||
_url = urlparse(url)
|
||||
url = f"{_url.scheme}://{_url.netloc}"
|
||||
if Node.select().where(Node.url == url).exists():
|
||||
flash("This node is already in the database.")
|
||||
else:
|
||||
flash("Seems like a valid node. Added to the database and will check soon.")
|
||||
node = Node(url=url)
|
||||
node.save()
|
||||
return redirect("/")
|
||||
|
||||
@app.cli.command("validate")
|
||||
def validate():
|
||||
nodes = Node.select().where(Node.validated == False)
|
||||
for node in nodes:
|
||||
now = datetime.now()
|
||||
is_onion = node.url.split(":")[1].endswith(".onion")
|
||||
logging.info(f"Attempting to validate {node.url}")
|
||||
if is_onion:
|
||||
logging.info("onion address found")
|
||||
node.tor = True
|
||||
try:
|
||||
r = requests.get(node.url + "/get_info", timeout=3)
|
||||
r.raise_for_status()
|
||||
assert "height" in r.json()
|
||||
assert "nettype" in r.json()
|
||||
nettype = r.json()["nettype"]
|
||||
logging.info("success")
|
||||
if nettype in ["mainnet", "stagenet", "testnet"]:
|
||||
node.nettype = nettype
|
||||
node.available = True
|
||||
node.validated = True
|
||||
node.datetime_checked = now
|
||||
node.save()
|
||||
else:
|
||||
logging.info("unexpected nettype")
|
||||
except requests.exceptions.ConnectTimeout:
|
||||
logging.info("connection timed out")
|
||||
node.delete_instance()
|
||||
except requests.exceptions.SSLError:
|
||||
logging.info("invalid certificate")
|
||||
node.delete_instance()
|
||||
except requests.exceptions.ConnectionError:
|
||||
logging.info("connection error")
|
||||
node.delete_instance()
|
||||
except requests.exceptions.HTTPError:
|
||||
logging.info("http error, 4xx or 5xx")
|
||||
node.delete_instance()
|
||||
except Exception as e:
|
||||
logging.info("failed for reasons unknown")
|
||||
node.delete_instance()
|
||||
|
||||
@app.route("/about")
|
||||
def about():
|
||||
|
|
|
|||
|
|
@ -0,0 +1,7 @@
|
|||
from flask_wtf import FlaskForm
|
||||
from wtforms import StringField
|
||||
from wtforms.validators import DataRequired
|
||||
|
||||
|
||||
class SubmitNode(FlaskForm):
|
||||
node_url = StringField('Node URL:', validators=[DataRequired()])
|
||||
|
|
@ -4,19 +4,17 @@ from xmrnodes import config
|
|||
|
||||
|
||||
data_dir = getattr(config, 'DATA_FOLDER', './data')
|
||||
db = SqliteDatabase(f"{data_dir}/db/sqlite.db")
|
||||
db = SqliteDatabase(f"{data_dir}/sqlite.db")
|
||||
|
||||
class Node(Model):
|
||||
id = AutoField()
|
||||
scheme = CharField()
|
||||
address = CharField()
|
||||
port = IntegerField()
|
||||
version = CharField(null=True)
|
||||
url = CharField()
|
||||
tor = BooleanField(default=False)
|
||||
available = BooleanField(default=False)
|
||||
mainnet = BooleanField(default=False)
|
||||
validated = BooleanField(default=False)
|
||||
nettype = CharField(null=True)
|
||||
datetime_entered = DateTimeField(default=datetime.now)
|
||||
datetime_checked = DateTimeField(default=datetime.now)
|
||||
datetime_checked = DateTimeField(default=None, null=True)
|
||||
datetime_failed = DateTimeField(default=None, null=True)
|
||||
|
||||
class Meta:
|
||||
|
|
|
|||
|
|
@ -9,13 +9,25 @@
|
|||
</div>
|
||||
|
||||
{% for node in nodes %}
|
||||
{{ node }}<br>
|
||||
{{ node.url }}<br>
|
||||
{% endfor %}
|
||||
|
||||
<form id="addnode" method="POST">
|
||||
<label>Node URL:</label>
|
||||
<input type="text" name="url"/>
|
||||
<input type="submit" value="Submit" name="submit" class="submit" id="submit" />
|
||||
<form method="POST" action="{{ url_for('add') }}">
|
||||
{{ form.csrf_token }}
|
||||
{% for f in form %}
|
||||
{% if f.name != 'csrf_token' %}
|
||||
<div class="form-group">
|
||||
{{ f.label }}
|
||||
{{ f }}
|
||||
</div>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
<ul>
|
||||
{% for field, errors in form.errors.items() %}
|
||||
<li>{{ form[field].label }}: {{ ', '.join(errors) }}</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
<input type="submit" value="Send" class="btn btn-link btn-outline btn-xl">
|
||||
</form>
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue