diff --git a/.env.sample b/.env.sample index b5c976c..b560b1c 100644 --- a/.env.sample +++ b/.env.sample @@ -1,2 +1,6 @@ -DB_URL=postgresql://postgres:postgres@localhost:5432/postgres DUNE_API_KEY= +DB_URL=postgresql://postgres:postgres@localhost:5432/postgres + +QUERY_ID= +POLL_FREQUENCY=10 +QUERY_ENGINE=medium diff --git a/Makefile b/Makefile index f62caf8..370879b 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -VENV = venv +VENV = .venv PYTHON = $(VENV)/bin/python3 PIP = $(VENV)/bin/pip PROJECT_ROOT = src @@ -25,7 +25,6 @@ lint: check: make fmt make lint - make types run: python -m src.main diff --git a/README.md b/README.md new file mode 100644 index 0000000..58fcd75 --- /dev/null +++ b/README.md @@ -0,0 +1,13 @@ +# Dune Sync V2 + + + + +## Local Development + +Fill out the empty fields in [Sample Env](.env.sample) (`DUNE_API_KEY` and `QUERY_ID`) + +```shell +docker-compose up -d +python -m src.main +``` diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 4a4038b..0000000 --- a/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -requests -pandas -sqlalchemy -python-dotenv -psycopg2-binary diff --git a/requirements/prod.txt b/requirements/prod.txt index a4e655b..eb06a5d 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -1,4 +1,4 @@ -requests +dune-client>=1.7.7 pandas sqlalchemy python-dotenv diff --git a/src/main.py b/src/main.py index b3b8702..bffcb05 100644 --- a/src/main.py +++ b/src/main.py @@ -1,18 +1,21 @@ import os -import requests import pandas as pd +from dune_client.client import DuneClient +from dune_client.query import QueryBase from sqlalchemy import create_engine from dotenv import load_dotenv from src.types import DUNE_TO_PG load_dotenv() -DUNE_API_KEY = os.getenv("DUNE_API_KEY") -DB_URL = os.getenv("DB_URL") -TABLE_NAME = "dune_data" +DUNE_API_KEY = os.environ.get("DUNE_API_KEY") +DB_URL = os.environ.get("DB_URL") -API_URL = "https://api.dune.com/api/v1/query/4132129/results" -HEADERS = {"x-dune-api-key": DUNE_API_KEY} +# TODO(bh2smith): parse config file for most of the following stuff +QUERY_ID = int(os.environ.get("QUERY_ID")) +POLL_FREQUENCY = int(os.environ.get("POLL_FREQUENCY")) +QUERY_ENGINE = os.environ.get("QUERY_ENGINE") +TABLE_NAME = f"dune_data_{QUERY_ID}" def reformat_varbinary_columns(df, varbinary_columns): @@ -22,23 +25,22 @@ def reformat_varbinary_columns(df, varbinary_columns): def fetch_dune_data(): - response = requests.get(API_URL, headers=HEADERS, timeout=10) - if response.status_code == 200: - data = response.json() - result = data["result"] - metadata = result["metadata"] - dtypes, varbinary_columns = {}, [] - for name, d_type in zip(metadata["column_names"], metadata["column_types"]): - dtypes[name] = DUNE_TO_PG[d_type] - if d_type == "varbinary": - varbinary_columns.append(name) - df = pd.DataFrame(result["rows"]) - # escape bytes - df = reformat_varbinary_columns(df, varbinary_columns) - return df, dtypes - - print(f"Error fetching data: {response.status_code}") - return None + result = ( + DuneClient(DUNE_API_KEY, performance=QUERY_ENGINE) + .run_query(query=QueryBase(QUERY_ID), ping_frequency=POLL_FREQUENCY) + .result + ) + + metadata = result.metadata + dtypes, varbinary_columns = {}, [] + for name, d_type in zip(metadata.column_names, metadata.column_types): + dtypes[name] = DUNE_TO_PG[d_type] + if d_type == "varbinary": + varbinary_columns.append(name) + df = pd.DataFrame(result.rows) + # escape bytes + df = reformat_varbinary_columns(df, varbinary_columns) + return df, dtypes def save_to_postgres(df, dtypes):