updating data and new cleaning script

data/all_trades_profitability.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1df952a693ba00cc0b11bca8ff4c6805415f2d006b3cd47242e43e7cdc7d5fe1
-size 3266876
+oid sha256:804722496702da46e6034175b54c73778fd4c5b7794d29967dccb2f2f6432603
+size 3290989

data/fpmmTrades.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:390f96495582e69ae82225a61e6473c1fe6536081b326a6bd11617be45ce672a
-size 10816943
+oid sha256:59bd9c58e15de9dcb4ae76cd8adca7750b460abfa2bdf79ee5042d3e3b5c396e
+size 13934569

data/fpmms.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3b5dcd19c7922e3f7168a139b0d63c335c921343faa15852b6ae04888f7e006a
-size 504817
+oid sha256:16b0570a9c07e0ef5d137ede96584fcfe1645a784a7380a83b9bdfa5829ad3e2
+size 515347

data/invalid_trades.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:30e0fa52d0c605961b5a12bec47bc3b0288b02b814c61cc7f8a33ad793f8bd30
-size 84013
+oid sha256:18f4b47e3c764b8c7f157b4b408d0c97e3436f58d86eb39edecf2a7cf2748a21
+size 84033

data/new_fpmmTrades.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84442b8ab800d01ec66e0c78efebfc31a5d954d3fdddfaf5fab41a75030a3967
+size 3267040

data/new_tools.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67da51ef342b56056a5bfb49f78ecab7354731e2cd88d16ccbffeaa141e175ec
+size 64443733

data/outliers.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3910fd14580aac1b02c49152dbc5fb7b282aaa52b81e3e634801bf673590c8fb
+oid sha256:9fa72b86e20493b5a02dff2cf9173d394546b5eaba1de21469bb66593f7939e1
 size 18274

data/summary_profitability.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a55a6c4c7ef5da8db27b61c268eccbd1d426c456a0d17efa4b22b7c69ed1454d
-size 78788
+oid sha256:522003ca4b76df815bf662ebc92478bc103652ac9f82dc82718578c26c650509
+size 87497

data/t_map.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7b67cf178943b82b5286b7c2adb6329e1e23fffce807ebf299684746813f55de
-size 22992649
+oid sha256:69d6fcd0360c5bbd646fa748b3f5a1e4bcccae358f32c85aa96509cdb6319c76
+size 24153722

data/tools.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7e4ace8d172836c379ee23bde678f19d9eeec28e7bd31bf9e95dc914ac5c9bc5
-size 407088092
+oid sha256:aa6d6407da787ae9d2ed80233d939f57feae82cd66a8193937b861c601f24828
+size 406224765

data/tools_accuracy.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f7a3622338d1eb2f23824031733ecdd77ae77eff7cb2b1c879aba05b0966d2cc
-size 1133
+oid sha256:b0113d5cbc3c5eb981b5f1c5a7776f616fc52f44b15b5f96880a16989fa07d16
+size 1240

scripts/cleaning_old_info.py

@@ -0,0 +1,66 @@
+import requests
+import json
+import pandas as pd
+import numpy as np
+from profitability import DATA_DIR, summary_analyse
+from staking import label_trades_by_staking
+
+
+def clean_old_data_from_parquet_files(cutoff_date: str):
+    # Convert the string to datetime64[ns, UTC]
+    min_date_utc = pd.to_datetime(cutoff_date, format="%Y-%m-%d", utc=True)
+
+    # clean tools.parquet
+    try:
+        tools = pd.read_parquet(DATA_DIR / "tools.parquet")
+
+        # make sure creator_address is in the columns
+        assert "trader_address" in tools.columns, "trader_address column not found"
+
+        # lowercase and strip creator_address
+        tools["trader_address"] = tools["trader_address"].str.lower().str.strip()
+
+        tools["request_time"] = pd.to_datetime(tools["request_time"], utc=True)
+
+        print(f"length before filtering {len(tools)}")
+        tools = tools.loc[tools["request_time"] > min_date_utc]
+        print(f"length after filtering {len(tools)}")
+        tools.to_parquet(DATA_DIR / "tools.parquet", index=False)
+
+    except Exception as e:
+        print(f"Error cleaning tools file {e}")
+
+    # clean all_trades_profitability.parquet
+    try:
+        all_trades = pd.read_parquet(DATA_DIR / "all_trades_profitability.parquet")
+
+        all_trades["creation_timestamp"] = pd.to_datetime(
+            all_trades["creation_timestamp"], utc=True
+        )
+
+        print(f"length before filtering {len(all_trades)}")
+        all_trades = all_trades.loc[all_trades["creation_timestamp"] > min_date_utc]
+        print(f"length after filtering {len(all_trades)}")
+
+    except Exception as e:
+        print(f"Error cleaning all trades profitability file {e}")
+
+    # generate again summary_profitability.parquet
+    try:
+        print("Summarising trades...")
+        summary_df = summary_analyse(all_trades)
+
+        # add staking labels
+        label_trades_by_staking(trades_df=all_trades, update=False)
+
+        # save to parquet
+        all_trades.to_parquet(
+            DATA_DIR / "all_trades_profitability.parquet", index=False
+        )
+        summary_df.to_parquet(DATA_DIR / "summary_profitability.parquet", index=False)
+    except Exception as e:
+        print(f"Error generating summary and saving all trades profitability file {e}")
+
+
+if __name__ == "__main__":
+    clean_old_data_from_parquet_files("2024-09-15")

scripts/get_mech_info.py

@@ -3,7 +3,6 @@ from typing import Any
 from datetime import datetime, timedelta, UTC
 from utils import SUBGRAPH_API_KEY, measure_execution_time, DATA_DIR
 import requests
-import json
 import pandas as pd
 import numpy as np
 from mech_request_utils import (
@@ -13,7 +12,6 @@ from mech_request_utils import (
     fix_duplicate_requestIds,
     merge_requests_delivers,
     get_ipfs_data,
-    only_delivers_loop,
     merge_json_files,
 )
 

scripts/profitability.py

@@ -40,7 +40,6 @@ from utils import (
     wei_to_unit,
     convert_hex_to_int,
     _to_content,
-    read_parquet_files,
     JSON_DATA_DIR,
 )
 from queries import omen_xdai_trades_query, conditional_tokens_gc_user_query
@@ -581,13 +580,19 @@ def run_profitability_analysis(
         update_tools_parquet(tools_filename)
         all_trades_df = update_all_trades_parquet(all_trades_df)
 
+    # debugging purposes
+    all_trades_df.to_parquet(JSON_DATA_DIR / "all_trades_df.parquets")
     # filter invalid markets. Condition: "is_invalid" is True
     invalid_trades = all_trades_df.loc[all_trades_df["is_invalid"] == True]
     if merge:
         try:
             old_invalid_trades = pd.read_parquet(DATA_DIR / "invalid_trades.parquet")
-            merge_df = pd.concat([old_invalid_trades, invalid_trades], ignore_index=True)
+            merge_df = pd.concat(
+                [old_invalid_trades, invalid_trades], ignore_index=True
+            )
             invalid_trades = merge_df.drop_duplicates()
+        except Exception as e:
+            print(f"Error updating the invalid trades parquet {e}")
     invalid_trades.to_parquet(DATA_DIR / "invalid_trades.parquet", index=False)
 
     all_trades_df = all_trades_df.loc[all_trades_df["is_invalid"] == False]

scripts/pull_data.py

@@ -124,7 +124,7 @@ def only_new_weekly_analysis():
     rpc = RPC
     # Run markets ETL
     logging.info("Running markets ETL")
-    mkt_etl(MARKETS_FILENAME)
+    # mkt_etl(MARKETS_FILENAME)
     logging.info("Markets ETL completed")
 
     # New tools ETL

scripts/staking.py

@@ -174,13 +174,16 @@ def get_trader_address_staking(trader_address: str, service_map: dict) -> str:
     return check_owner_staking_contract(owner_address=owner)
 
 
-def label_trades_by_staking(trades_df: pd.DataFrame) -> pd.DataFrame:
+def label_trades_by_staking(
+    trades_df: pd.DataFrame, update: bool = True
+) -> pd.DataFrame:
     with open(DATA_DIR / "service_map.pkl", "rb") as f:
         service_map = pickle.load(f)
     # get the last service id
     keys = service_map.keys()
     last_key = max(keys)
-    update_service_map(start=last_key)
+    if update:
+        update_service_map(start=last_key)
     all_traders = trades_df.trader_address.unique()
     trades_df["staking"] = ""
     for trader in tqdm(all_traders, desc="Labeling traders by staking", unit="trader"):

scripts/tools.py

@@ -98,7 +98,8 @@ HTTP_TIMEOUT = 10
 N_IPFS_RETRIES = 1
 N_RPC_RETRIES = 100
 RPC_POLL_INTERVAL = 0.05
-IPFS_POLL_INTERVAL = 0.05
+# IPFS_POLL_INTERVAL = 0.05  # low speed
+IPFS_POLL_INTERVAL = 0.2  # high speed
 IRRELEVANT_TOOLS = [
     "openai-text-davinci-002",
     "openai-text-davinci-003",
@@ -585,7 +586,7 @@ def parse_store_json_events_parallel(json_events: Dict[str, Any], output_filenam
             contents.append(current_mech_contents)
 
     tools = pd.concat(contents, ignore_index=True)
-    print(f"Adding market creators info. Length of the tools file = {tools}")
+    print(f"Adding market creators info. Length of the tools file = {len(tools)}")
     tools = add_market_creator(tools)
     print(
         f"Length of the tools dataframe after adding market creators info= {len(tools)}"