45 lines
1.2 KiB
Python
45 lines
1.2 KiB
Python
import httpx
|
|
import time
|
|
import modules
|
|
import os
|
|
from dotenv import load_dotenv
|
|
from supabase import create_client, Client
|
|
load_dotenv()
|
|
url: str = os.environ.get("SUPABASE_URL")
|
|
key: str = os.environ.get("SUPABASE_KEY")
|
|
supabase: Client = create_client(url, key)
|
|
|
|
|
|
def scraper():
|
|
transport = httpx.HTTPTransport(retries=5)
|
|
client = httpx.Client(transport=transport)
|
|
|
|
api_list = client.get("https://suchwow.xyz/api/list", timeout=100).json()
|
|
api_list.reverse()
|
|
|
|
for item in api_list:
|
|
try:
|
|
supabase.table("suchmeme").insert(item).execute()
|
|
print(f'Inserted meme {item["id"]} to Supabase DB')
|
|
post(item)
|
|
except Exception as e:
|
|
if 'duplicate key value violates unique constraint' in str(e):
|
|
print("Meme already exists")
|
|
continue
|
|
else:
|
|
print(f'Error upon inserting data in DB with meme {item["id"]} due to {e}')
|
|
continue
|
|
|
|
|
|
def post(item):
|
|
modules.masto(item)
|
|
modules.tumblr(item)
|
|
|
|
|
|
while True:
|
|
try:
|
|
scraper()
|
|
except Exception as e:
|
|
print(e)
|
|
print("Database updated")
|
|
time.sleep(120) |