diff --git a/Makefile b/Makefile index 94691a4..3108acf 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,7 @@ +setup: + python3 -m venv .venv + .venv/bin/pip install -r requirements.txt + up: docker-compose up -d diff --git a/requirements.txt b/requirements.txt index eb559c5..3798115 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,6 @@ peewee gunicorn arrow flask_wtf +pysocks git+https://github.com/cdiv1e12/py-levin +geoip2 diff --git a/xmrnodes/app.py b/xmrnodes/app.py index 844b797..fcc6264 100644 --- a/xmrnodes/app.py +++ b/xmrnodes/app.py @@ -1,18 +1,23 @@ -import arrow + import json -import requests import re import logging -import click from os import makedirs from random import shuffle +from socket import gethostbyname_ex from datetime import datetime, timedelta -from flask import Flask, request, redirect + +import geoip2.database +import arrow +import requests +import click +from flask import Flask, request, redirect, jsonify from flask import render_template, flash, url_for from urllib.parse import urlparse + from xmrnodes.helpers import determine_crypto, is_onion, make_request, retrieve_peers from xmrnodes.forms import SubmitNode -from xmrnodes.models import Node, HealthCheck +from xmrnodes.models import Node, HealthCheck, Peer from xmrnodes import config @@ -54,6 +59,68 @@ def index(): form=form ) +@app.route("/nodes.json") +def nodes_json(): + nodes = Node.select().where( + Node.validated==True + ).where( + Node.nettype=="mainnet" + ) + xmr_nodes = [n for n in nodes if n.crypto == "monero"] + wow_nodes = [n for n in nodes if n.crypto == "wownero"] + return jsonify({ + "monero": { + "clear": [n.url for n in xmr_nodes if n.is_tor == False], + "onion": [n.url for n in xmr_nodes if n.is_tor == True] + }, + "wownero": { + "clear": [n.url for n in wow_nodes if n.is_tor == False], + "onion": [n.url for n in wow_nodes if n.is_tor == True] + } + }) + +@app.route("/wow_nodes.json") +def wow_nodes_json(): + nodes = Node.select().where( + Node.validated==True + ).where( + Node.nettype=="mainnet" + ).where( + Node.crypto=="wownero" + ) + nodes = [n for n in nodes] + return jsonify({ + "clear": [n.url for n in nodes if n.is_tor == False], + "onion": [n.url for n in nodes if n.is_tor == True] + }) + +@app.route("/map") +def map(): + peers = Peer.select() + nodes = list() + _nodes = Node.select().where( + Node.is_tor == False, + Node.crypto == 'monero', + Node.validated == True, + Node.nettype == 'mainnet' + ) + with geoip2.database.Reader('./data/GeoLite2-City.mmdb') as reader: + for node in _nodes: + try: + _url = urlparse(node.url) + ip = gethostbyname_ex(_url.hostname)[2][0] + response = reader.city(ip) + nodes.append((response.location.longitude, response.location.latitude, _url.hostname, node.datetime_entered)) + except: + pass + + return render_template( + "map.html", + peers=peers, + nodes=nodes, + source_node=config.NODE_HOST + ) + @app.route("/resources") def resources(): return render_template("resources.html") @@ -123,8 +190,56 @@ def check(): @app.cli.command("get_peers") def get_peers(): - r = retrieve_peers() - print(r) + all_peers = [] + print(f'[+] Retrieving initial peers from {config.NODE_HOST}:{config.NODE_PORT}') + initial_peers = retrieve_peers(config.NODE_HOST, config.NODE_PORT) + with geoip2.database.Reader('./data/GeoLite2-City.mmdb') as reader: + for peer in initial_peers: + if peer not in all_peers: + all_peers.append(peer) + _url = urlparse(peer) + url = f"{_url.scheme}://{_url.netloc}".lower() + if not Peer.select().where(Peer.url == peer).exists(): + response = reader.city(_url.hostname) + p = Peer( + url=peer, + country=response.country.name, + city=response.city.name, + postal=response.postal.code, + lat=response.location.latitude, + lon=response.location.longitude, + ) + p.save() + print(f'{peer} - saving new peer') + else: + print(f'{peer} - already seen') + + try: + print(f'[+] Retrieving crawled peers from {_url.netloc}') + new_peers = retrieve_peers(_url.hostname, _url.port) + for peer in new_peers: + all_peers.append(peer) + _url = urlparse(peer) + url = f"{_url.scheme}://{_url.netloc}".lower() + if not Peer.select().where(Peer.url == peer).exists(): + response = reader.city(_url.hostname) + p = Peer( + url=peer, + country=response.country.name, + city=response.city.name, + postal=response.postal.code, + lat=response.location.latitude, + lon=response.location.longitude, + ) + p.save() + print(f'{peer} - saving new peer') + else: + print(f'{peer} - already seen') + except: + pass + + print(f'{len(all_peers)} peers found from {config.NODE_HOST}:{config.NODE_PORT}') + @app.cli.command("validate") def validate(): @@ -182,7 +297,7 @@ def export(): def import_(): all_nodes = [] export_dir = f"{config.DATA_DIR}/export.txt" - with open(export_dir, 'r') as f: + with open(export_dir, "r") as f: for url in f.readlines(): try: n = url.rstrip().lower() diff --git a/xmrnodes/helpers.py b/xmrnodes/helpers.py index b86ef38..91dfdfa 100644 --- a/xmrnodes/helpers.py +++ b/xmrnodes/helpers.py @@ -54,12 +54,14 @@ def is_onion(url: str): else: return False -def retrieve_peers(): +def retrieve_peers(host, port): try: + print(f'[.] Connecting to {host}:{port}') sock = socket.socket() - sock.connect((config.NODE_HOST, int(config.NODE_PORT)) + sock.settimeout(5) + sock.connect((host, int(port))) except: - sys.stderr.write("unable to connect to %s:%d\n" % (config.NODE_HOST, int(config.NODE_PORT)) + sys.stderr.write("unable to connect to %s:%d\n" % (host, int([port]))) sys.exit() bucket = Bucket.create_handshake_request() @@ -84,9 +86,9 @@ def retrieve_peers(): buckets.append(bucket) if bucket.command == 1001: - peers = bucket.get_peers() or [] + _peers = bucket.get_peers() or [] - for peer in peers: + for peer in _peers: try: peers.append('http://%s:%d' % (peer['ip'].ip, peer['port'].value)) except: diff --git a/xmrnodes/models.py b/xmrnodes/models.py index 1c527ea..a6b21ca 100644 --- a/xmrnodes/models.py +++ b/xmrnodes/models.py @@ -1,5 +1,8 @@ -from peewee import * +from urllib.parse import urlparse from datetime import datetime + +from peewee import * + from xmrnodes import config @@ -22,6 +25,22 @@ class Node(Model): class Meta: database = db +class Peer(Model): + id = AutoField() + url = CharField(unique=True) + country = CharField(null=True) + city = CharField(null=True) + postal = IntegerField(null=True) + lat = FloatField(null=True) + lon = FloatField(null=True) + datetime = DateTimeField(default=datetime.utcnow) + + def get_ip(self): + return urlparse(self.url).hostname + + class Meta: + database = db + class HealthCheck(Model): id = AutoField() node = ForeignKeyField(Node, backref='healthchecks') @@ -31,4 +50,4 @@ class HealthCheck(Model): class Meta: database = db -db.create_tables([Node, HealthCheck]) +db.create_tables([Node, HealthCheck, Peer]) diff --git a/xmrnodes/templates/base.html b/xmrnodes/templates/base.html index dc299e3..8d0c231 100644 --- a/xmrnodes/templates/base.html +++ b/xmrnodes/templates/base.html @@ -45,6 +45,8 @@
Contact me - + Map + - Source Code - Resources diff --git a/xmrnodes/templates/map.html b/xmrnodes/templates/map.html new file mode 100644 index 0000000..c4e8415 --- /dev/null +++ b/xmrnodes/templates/map.html @@ -0,0 +1,190 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + XMR Nodes + + + + +{% with messages = get_flashed_messages() %} + {% if messages %} + + {% endif %} +{% endwith %} + +
+
+ Contact me + - + Source Code + - + Resources +
+ + +

View Map

+
+ +

Found Peers (via source node, levin p2p): {{ peers | length }}

+

Added Nodes (Monero mainnet): {{ nodes | length }}

+

Source Node: {{ source_node }}

+

+ This is not a full representation of the entire Monero network, + just a look into the peers being crawled from the source node ({{ source_node }}) + and the nodes already added to the monero.fail database. + New peers are searched for on a recurring interval throughout the day. +

+ + + + + + + +