From 694b052d13ca4a7d218f5bfe6cd80af9fc678054 Mon Sep 17 00:00:00 2001 From: ShootInPutIn007 Date: Wed, 1 May 2024 14:26:50 +0200 Subject: [PATCH] Changing the backend to query address labels and ens datatables instead of relying on the flipside api --- .env.sample | 4 +-- label.py | 80 ++++++++++++++++++++++-------------------------- requirements.txt | 1 - simulate.py | 8 ++--- 4 files changed, 42 insertions(+), 51 deletions(-) diff --git a/.env.sample b/.env.sample index 8645377..1c76887 100644 --- a/.env.sample +++ b/.env.sample @@ -11,8 +11,8 @@ GCS_BUCKET_NAME= GOOGLE_SHEET_ID= GOOGLE_WORKSHEET_NAME= API_TOKEN= -FLIPSIDE_API_KEY= -FLIPSIDE_ENDPOINT_URL=https://api-v2.flipsidecrypto.xyz +ENS_BIGQUERY_DATATABLE= +LABELS_BIGQUERY_DATATABLE= DEFAULT_MODEL=claude-3-haiku-20240307 # 'claude-3-haiku-20240307' or 'claude-3-sonnet-20240229' or 'claude-3-opus-20240229' CORS_ALLOWED_ORIGINS= PORT=5000 diff --git a/label.py b/label.py index bd5e969..ed9042d 100644 --- a/label.py +++ b/label.py @@ -1,9 +1,7 @@ import json import re -from flipside import Flipside import pandas as pd - # Recursively iterate over the json object looking for specified pattern def explore_json(obj, items, pattern): try: @@ -40,64 +38,60 @@ def format (addresses): return addresses_str -# Query Flipside -def query_flipside (addresses_str, endpoint): - sql = f""" - select address, - address_name, - label, - label_type, - label_subtype - from ethereum.core.dim_labels - where lower(address) in ({addresses_str}) - """ - - query_result_set = endpoint.query(sql) - df = pd.DataFrame(query_result_set) - return df - -# Put results into a json object -def to_json(df): +# Query labels datatables +def query_labels (addresses_str, labels_dataset, ens_dataset, client): try: - json_data = [] - data = df[1][4] - if data is None: - print("Data is None, returning an empty JSON object") - return json.dumps({'address_labels': []}, indent=4) + sql = f""" + with labels_data as ( + select address, + address_name + from {labels_dataset} + where address in ({addresses_str}) + ), + ens_data as ( + select address, + tag as address_name + from {ens_dataset} + where address in ({addresses_str}) + and address not in (select address from labels_data) + ), + merged_data as ( + select * from labels_data + union all + select * from ens_data + ) - for index, item in enumerate(data): - json_data.append({ - 'address': item[0], - 'address_name': item[1], - 'label': item[2], - 'label_type': item[3], - 'label_subtype': item[4] - }) - - json_object = json.dumps({'address_labels': json_data}, indent=4) - return json_object + select * from merged_data + """ + df = client.query(sql).result().to_dataframe() + return df except Exception as e: - print("Error at to_json: ", e) + print("Error at query_labels: ", e) + return pd.DataFrame() # Fetch address labels -def fetch_address_labels(sim_data, endpoint): +def fetch_address_labels(sim_data, labels_dataset, ens_dataset, client): address_regex = r'^0x[0-9a-fA-F]{40}$' try: addresses = extract(sim_data, address_regex) addresses_str = format(addresses) - labels_data = query_flipside(addresses_str,endpoint) - labels_json = to_json(labels_data) + + labels = query_labels (addresses_str, labels_dataset, ens_dataset, client) + + labels_json = labels.to_json(orient='records') + labels_json = '{"address_labels": ' + labels_json + '}' + return labels_json except Exception as e: print("Error at fetch_address_labels: ", e) # Add address labels to the original sim_data -def add_labels(sim_data, endpoint): +def add_labels(sim_data, labels_dataset, ens_dataset, client): try: - labels_json = json.loads(fetch_address_labels(sim_data, endpoint)) + labels_json = json.loads(fetch_address_labels(sim_data, labels_dataset, ens_dataset, client)) for key, value in labels_json.items(): sim_data[key] = value - + return sim_data except Exception as e: print("Error at add_labels: ", e) diff --git a/requirements.txt b/requirements.txt index a36eaeb..86c28a0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,6 @@ asyncio aiohttp fastapi==0.110.1 fastapi_limiter==0.1.6 -flipside==2.0.8 starlette==0.37.2 redis==5.1.0b4 uvicorn==0.13.1 diff --git a/simulate.py b/simulate.py index f4cc6fb..0dfa858 100644 --- a/simulate.py +++ b/simulate.py @@ -9,7 +9,6 @@ from dotenv import load_dotenv from web3 import Web3, AsyncWeb3 import decimal -from flipside import Flipside from label import add_labels w3 = AsyncWeb3(AsyncWeb3.AsyncHTTPProvider('https://cloudflare-eth.com')) @@ -330,9 +329,8 @@ async def simulate_transaction(tx_hash, block_number, from_address, to_address, tenderly_account_slug = os.getenv('TENDERLY_ACCOUNT_SLUG') tenderly_project_slug = os.getenv('TENDERLY_PROJECT_SLUG') tenderly_access_key = os.getenv('TENDERLY_ACCESS_KEY') - flipside_api_key = os.getenv('FLIPSIDE_API_KEY') - flipside_endpoint_url = os.getenv('FLIPSIDE_ENDPOINT_URL') - flipside = Flipside(flipside_api_key, flipside_endpoint_url) + labels_dataset = os.getenv('LABELS_BIGQUERY_DATATABLE') + ens_dataset = os.getenv('ENS_BIGQUERY_DATATABLE') tx_details = { 'network_id': NETWORK_CONFIGS[network]['network_id'], @@ -370,7 +368,7 @@ async def simulate_transaction(tx_hash, block_number, from_address, to_address, trimmed_initial = await extract_useful_fields(sim_data) trimmed_decimals = await apply_decimals(trimmed_initial) trimmed_logs_applied= await apply_logs(trimmed_decimals) - trimmed= add_labels(trimmed_logs_applied, flipside) + trimmed = add_labels(trimmed_logs_applied, labels_dataset, ens_dataset, bigquery_client) try: blob = bucket.blob(f'{network}/transactions/simulations/trimmed/{tx_hash}.json') blob.upload_from_string(json.dumps(trimmed))