From 4ae577a648044b7ec46af8f28a57ffa4b8557948 Mon Sep 17 00:00:00 2001 From: lza_menace Date: Tue, 11 May 2021 20:38:29 -0700 Subject: [PATCH] init --- .gitignore | 3 +++ Dockerfiles/files/setup_es.py | 51 +++++++++++++++++++++++++++++++++++ Dockerfiles/setup | 5 ++++ Makefile | 36 +++++++++++++++++++++++++ docker-compose.yaml | 28 +++++++++++++++++++ main.py | 51 +++++++++++++++++++++++++++++++++++ requirements.txt | 9 +++++++ 7 files changed, 183 insertions(+) create mode 100644 .gitignore create mode 100644 Dockerfiles/files/setup_es.py create mode 100644 Dockerfiles/setup create mode 100644 Makefile create mode 100644 docker-compose.yaml create mode 100644 main.py create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d713c78 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/data +.idea +.venv diff --git a/Dockerfiles/files/setup_es.py b/Dockerfiles/files/setup_es.py new file mode 100644 index 0000000..2522a07 --- /dev/null +++ b/Dockerfiles/files/setup_es.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +import requests +import json +from time import sleep + +es_headers = {'Content-Type': 'application/json'} +kibana_headers = {'Content-Type': 'application/json', 'kbn-xsrf': 'true'} + +nginx_pattern = { + "index_patterns": ["wownero-block-data-*"], + "mappings": { + "properties": { + "datetime": { + "type": "date", + "format": "yyyy-MM-dd HH:mm:ss" + } + } + } +} + +remove_low_disk = { + "transient": { + "cluster.routing.allocation.disk.watermark.low": "30mb", + "cluster.routing.allocation.disk.watermark.high": "20mb", + "cluster.routing.allocation.disk.watermark.flood_stage": "10mb", + "cluster.info.update.interval": "1m" + } +} + +def wait_for_es(): + keep_trying = True + while keep_trying: + try: + requests.get('http://elasticsearch:9200/_cat/health', headers=es_headers) + keep_trying = False + except: + print('[!] Elasticsearch not ready yet....waiting') + sleep(15) + +def load_index_settings(): + # Publish index mappings and settings + print('[+] Adding index mapping and cluster settings') + r1 = requests.put('http://elasticsearch:9200/_template/wownero-block-data', headers=es_headers, data=json.dumps(nginx_pattern)) + r2 = requests.put('http://elasticsearch:9200/_cluster/settings', headers=es_headers, data=json.dumps(remove_low_disk)) + print(r1) + print(r2) + +if __name__ == '__main__': + wait_for_es() + load_index_settings() diff --git a/Dockerfiles/setup b/Dockerfiles/setup new file mode 100644 index 0000000..5891c51 --- /dev/null +++ b/Dockerfiles/setup @@ -0,0 +1,5 @@ +FROM python:3 +RUN python3 -m pip install requests +WORKDIR /srv +COPY ./files/setup_es.py . +ENTRYPOINT ["python3", "-u", "setup_es.py"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ac44272 --- /dev/null +++ b/Makefile @@ -0,0 +1,36 @@ +.PHONY: format help + +# Help system from https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html +.DEFAULT_GOAL := help + +help: + @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' + +setup: ## Install Python dependencies + python3 -m venv .venv + .venv/bin/pip install -r requirements.txt + +up: ## Run all the containers in the background + mkdir -p data/logs + docker-compose up -d --build + +run: ## Run Python script to fetch blocks and update Elasticsearch + .venv/bin/python3 main.py + +ps: ## Show currently running containers + docker-compose ps + +logs: ## Begin streaming logs to terminal + docker-compose logs -f + +stop: ## Stop all containers + docker-compose stop + +start: ## Start all containers + docker-compose start + +down: ## Stop and remove all containers + docker-compose down --remove-orphans + +clean: ## Stop and remove all containers and their data + rm -rf ./data/elasticsearch ./data/logstash/ diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..8c336c3 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,28 @@ +version: '3' +services: + setup: + build: + context: ./Dockerfiles + dockerfile: setup + kibana: + image: docker.elastic.co/kibana/kibana:7.1.0 + ports: + - 5601:5601 + environment: + ELASTICSEARCH_HOSTS: http://elasticsearch:9200 + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:7.1.0 + environment: + - node.name=es01 + - cluster.name=wownero-network + - discovery.type=single-node + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + ports: + - 9200:9200 + volumes: + - ./data/elasticsearch:/usr/share/elasticsearch/data diff --git a/main.py b/main.py new file mode 100644 index 0000000..9e2d917 --- /dev/null +++ b/main.py @@ -0,0 +1,51 @@ +import requests +import os +import arrow +import json + +payload = {'jsonrpc': '2.0', 'id': '0', 'method': '', 'params': {}} +headers = {'Content-Type': 'application/json'} +daemon = 'http://crypto.int.lzahq.tech:34568' +json_rpc = daemon + '/json_rpc' +data_path = './data/logs' + +def make_payload(rpc_method, params={}): + payload['method'] = rpc_method + payload['params'] = params + return payload + +def make_rpc(rpc_method, http_method='get', _payload={}): + if http_method == 'get': + req = requests.get + else: + req = requests.post + r = req( + json_rpc, timeout=5, headers=headers, + json=make_payload(rpc_method, _payload) + ) + r.raise_for_status() + return r.json()['result'] + +def run(): + top_block = make_rpc('get_block_count', 'get')['count'] + for block in range(top_block - 1, 0, -1): + _path = f'{data_path}/{block}.json' + if not os.path.isfile(_path): + block_data = make_rpc('get_block', 'get', {'height': block})['block_header'] + ts = arrow.get(block_data['timestamp']) + if block_data['timestamp'] == 0: + ts = arrow.get('2018-04-01 04:20:00') + block_data['datetime'] = ts.format('YYYY-MM-DD HH:mm:ss') + index_date = ts.format('YYYY-MM') + index_name = f'wownero-block-data-{index_date}' + r = requests.post( + f'http://localhost:9200/{index_name}/_doc/{block_data["height"]}', + json=block_data, timeout=5, headers=headers + ) + r.raise_for_status() + with open(_path, 'w') as f: + f.write(json.dumps(block_data)) + print(f'Added file {_path} for index {index_name}') + +if __name__ == '__main__': + run() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ee4e5a3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +arrow==1.1.0 +certifi==2020.12.5 +chardet==4.0.0 +idna==2.10 +python-dateutil==2.8.1 +requests==2.25.1 +six==1.16.0 +typing-extensions==3.10.0.0 +urllib3==1.26.4