updating all files with new mech calls functions and better trader type description

app.py

@@ -223,7 +223,7 @@ with demo:
                         trades_df=trades_df,
                     )
                 with gr.Column(scale=1):
-                    trade_details_text = get_trade_metrics_text()
+                    trade_details_text = get_trade_metrics_text(trader_type=None)
 
             trade_details_selector.change(
                 update_trade_details,
@@ -251,7 +251,7 @@ with demo:
                         trader_filter="Olas",
                     )
                 with gr.Column(scale=1):
-                    trade_details_text = get_trade_metrics_text()
+                    trade_details_text = get_trade_metrics_text(trader_type="Olas")
 
             def update_a_trade_details(trade_detail, trade_o_details_plot):
                 new_a_plot = plot_trade_metrics(
@@ -287,7 +287,7 @@ with demo:
                         trader_filter="non_Olas",
                     )
                 with gr.Column(scale=1):
-                    trade_details_text = get_trade_metrics_text()
+                    trade_details_text = get_trade_metrics_text("non_Olas")
 
             def update_na_trade_details(trade_detail, trade_details_plot):
                 new_no_plot = plot_trade_metrics(
@@ -306,7 +306,7 @@ with demo:
             if len(unknown_trades) > 0:
                 with gr.Row():
                     gr.Markdown(
-                        "# Weekly trading metrics for trades coming from unknown traders"
+                        "# Weekly trading metrics for trades coming from Unclassified traders"
                     )
                 with gr.Row():
                     trade_u_details_selector = gr.Dropdown(
@@ -323,7 +323,9 @@ with demo:
                             trader_filter="all",
                         )
                     with gr.Column(scale=1):
-                        trade_details_text = get_trade_metrics_text()
+                        trade_details_text = get_trade_metrics_text(
+                            trader_type="unclassified"
+                        )
 
                 def update_na_trade_details(trade_detail, trade_u_details_plot):
                     new_u_plot = plot_trade_metrics(

data/all_trades_profitability.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6d097ed0d81bd0fa9f2e301a1db66da1a5cb122f1ad8626327715dcaff127b83
-size 3558217
+oid sha256:ec1f80e9de64d8981ac58b91de9e17f371c0620544c9012168519a6a789b512c
+size 3537818

data/daily_info.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d6d36bd873a5fe9564f556eda0210078a0e9c3d0dd81fec2158cf029ed462573
-size 911119
+oid sha256:fed76273653048f900faca2d612b07f42be43d076238f0dac7f30e8882a1ec1b
+size 374565

data/service_map.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:574e816f13aee41f153346e0590bbaeb5115578faa6aa9129dbda8b49b4d7fd2
-size 90733
+oid sha256:6109116eb0c946088a55420b04cc85576985cb0bef7ec47c3b2be97ee85688e8
+size 90766

data/tools_accuracy.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:15077d12895fc6ee9f9e8e1ac3a1d84e45350e261590bb15af5a11378ce83c49
+oid sha256:33bf015940a44f02ababb579398272ffc258a48d10e16be075179f18f4a2d578
 size 1101

scripts/daily_data.py

@@ -5,7 +5,11 @@ from profitability import (
     label_trades_by_staking,
 )
 import pandas as pd
-from nr_mech_calls import create_unknown_traders_df
+from nr_mech_calls import (
+    create_unknown_traders_df,
+    compute_daily_mech_calls,
+    transform_to_datetime,
+)
 
 logging.basicConfig(level=logging.INFO)
 
@@ -16,8 +20,16 @@ def prepare_live_metrics(
 ):
     fpmmTrades = pd.read_parquet(TMP_DIR / trades_filename)
     tools = pd.read_parquet(TMP_DIR / tools_filename)
+
+    fpmmTrades["creationTimestamp"] = fpmmTrades["creationTimestamp"].apply(
+        lambda x: transform_to_datetime(x)
+    )
+    print("Computing the estimated mech calls dataset")
+    trader_mech_calls = compute_daily_mech_calls(fpmmTrades=fpmmTrades, tools=tools)
     print("Analysing trades...")
-    all_trades_df = analyse_all_traders(fpmmTrades, tools, daily_info=True)
+    all_trades_df = analyse_all_traders(
+        fpmmTrades, tools, trader_mech_calls, daily_info=True
+    )
 
     # staking label
     label_trades_by_staking(all_trades_df)

scripts/get_mech_info.py

@@ -1,7 +1,13 @@
 from string import Template
 from typing import Any
 from datetime import datetime, timedelta, UTC
-from utils import SUBGRAPH_API_KEY, measure_execution_time, DATA_DIR, TMP_DIR
+from utils import (
+    SUBGRAPH_API_KEY,
+    measure_execution_time,
+    DATA_DIR,
+    TMP_DIR,
+    NETWORK_SUBGRAPH_URL,
+)
 import requests
 import pandas as pd
 import numpy as np
@@ -16,14 +22,6 @@ from mech_request_utils import (
 )
 from web3_utils import updating_timestamps
 
-OLD_MECH_SUBGRAPH_URL = (
-    "https://api.thegraph.com/subgraphs/name/stakewise/ethereum-gnosis"
-)
-
-NETWORK_SUBGRAPH_URL = Template(
-    """https://gateway-arbitrum.network.thegraph.com/api/${subgraph_api_key}/subgraphs/id/FxV6YUix58SpYmLBwc9gEHkwjfkqwe1X5FJQjn8nKPyA"""
-)
-
 SUBGRAPH_HEADERS = {
     "Accept": "application/json, multipart/mixed",
     "Content-Type": "application/json",
@@ -321,18 +319,19 @@ def get_mech_events_since_last_run():
     last_block_number = get_last_block_number()
 
     # mech requests
-    requests_dict, duplicatedReqId = collect_all_mech_requests(
+    requests_dict, duplicatedReqId, nr_errors = collect_all_mech_requests(
         from_block=last_run_block_number,
         to_block=last_block_number,
         filename="new_mech_requests.json",
     )
-
+    print(f"NUMBER OF MECH REQUEST ERRORS={nr_errors}")
     # mech delivers
-    delivers_dict, duplicatedIds = collect_all_mech_delivers(
+    delivers_dict, duplicatedIds, nr_errors = collect_all_mech_delivers(
         from_block=last_run_block_number,
         to_block=last_block_number,
         filename="new_mech_delivers.json",
     )
+    print(f"NUMBER OF MECH DELIVER ERRORS={nr_errors}")
     if delivers_dict is None:
         return None
     # clean delivers
@@ -357,14 +356,14 @@ def get_mech_events_last_60_days():
     earliest_block_number = get_last_60_days_block_number()
     last_block_number = get_last_block_number()
     # mech requests
-    requests_dict, duplicatedReqId = collect_all_mech_requests(
+    requests_dict, duplicatedReqId, nr_errors = collect_all_mech_requests(
         from_block=earliest_block_number,
         to_block=last_block_number,
         filename="mech_requests.json",
     )
 
     # mech delivers
-    delivers_dict, duplicatedIds = collect_all_mech_delivers(
+    delivers_dict, duplicatedIds, nr_errors = collect_all_mech_delivers(
         from_block=earliest_block_number,
         to_block=last_block_number,
         filename="mech_delivers.json",

scripts/markets.py

@@ -26,7 +26,12 @@ import requests
 from tqdm import tqdm
 from typing import List, Dict
 from utils import SUBGRAPH_API_KEY, DATA_DIR, TMP_DIR
-from web3_utils import FPMM_QS_CREATOR, FPMM_PEARL_CREATOR, query_omen_xdai_subgraph
+from web3_utils import (
+    FPMM_QS_CREATOR,
+    FPMM_PEARL_CREATOR,
+    query_omen_xdai_subgraph,
+    OMEN_SUBGRAPH_URL,
+)
 from queries import (
     FPMMS_QUERY,
     ID_FIELD,
@@ -45,9 +50,6 @@ SubgraphResponseType = Dict[str, ResponseItemType]
 BATCH_SIZE = 1000
 DEFAULT_TO_TIMESTAMP = 2147483647  # around year 2038
 DEFAULT_FROM_TIMESTAMP = 0
-OMEN_SUBGRAPH_URL = Template(
-    """https://gateway-arbitrum.network.thegraph.com/api/${subgraph_api_key}/subgraphs/id/9fUVQpFwzpdWS9bq5WkAnmKbNNcoBwatMR4yZq81pbbz"""
-)
 
 MAX_UINT_HEX = "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
 DEFAULT_FILENAME = "fpmms.parquet"

scripts/mech_request_utils.py

@@ -36,7 +36,7 @@ from tools import (
 from tqdm import tqdm
 from web3_utils import FPMM_QS_CREATOR, FPMM_PEARL_CREATOR, IPFS_POLL_INTERVAL
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from utils import DATA_DIR, JSON_DATA_DIR
+from utils import DATA_DIR, JSON_DATA_DIR, MECH_SUBGRAPH_URL, SUBGRAPH_API_KEY
 
 NUM_WORKERS = 10
 BLOCKS_CHUNK_SIZE = 10000
@@ -44,7 +44,6 @@ TEXT_ALIGNMENT = 30
 MINIMUM_WRITE_FILE_DELAY_SECONDS = 20
 MECH_FROM_BLOCK_RANGE = 50000
 IPFS_ADDRESS = "https://gateway.autonolas.tech/ipfs/"
-THEGRAPH_ENDPOINT = "https://api.studio.thegraph.com/query/57238/mech/0.0.2"
 last_write_time = 0.0
 
 REQUESTS_QUERY_FILTER = """
@@ -109,10 +108,12 @@ def collect_all_mech_requests(from_block: int, to_block: int, filename: str) ->
     print(f"Fetching all mech requests from {from_block} to {to_block}")
     mech_requests = {}
     duplicated_reqIds = []
-    transport = RequestsHTTPTransport(url=THEGRAPH_ENDPOINT)
+    mech_subgraph_url = MECH_SUBGRAPH_URL.substitute(subgraph_api_key=SUBGRAPH_API_KEY)
+    transport = RequestsHTTPTransport(url=mech_subgraph_url)
     client = Client(transport=transport, fetch_schema_from_transport=True)
 
     id_gt = "0x00"
+    nr_errors = 0
     while True:
         variables = {
             "sender_not_in": [FPMM_QS_CREATOR, FPMM_PEARL_CREATOR],
@@ -121,9 +122,6 @@ def collect_all_mech_requests(from_block: int, to_block: int, filename: str) ->
             "blockNumber_lte": str(to_block),  # str
         }
         try:
-            # response = client.execute(
-            #     gql(REQUESTS_QUERY_FILTER), variable_values=variables
-            # )
             response = fetch_with_retry(client, REQUESTS_QUERY_FILTER, variables)
 
             items = response.get("requests", [])
@@ -137,6 +135,8 @@ def collect_all_mech_requests(from_block: int, to_block: int, filename: str) ->
                 else:
                     duplicated_reqIds.append(mech_request["id"])
         except Exception as e:
+            # counter for errors
+            nr_errors += 1
             print(f"Error while getting the response: {e}")
 
         id_gt = items[-1]["id"]
@@ -149,7 +149,7 @@ def collect_all_mech_requests(from_block: int, to_block: int, filename: str) ->
     print(f"Number of requests = {len(mech_requests)}")
     print(f"Number of duplicated req Ids = {len(duplicated_reqIds)}")
     save_json_file(mech_requests, filename)
-    return mech_requests, duplicated_reqIds
+    return mech_requests, duplicated_reqIds, nr_errors
 
 
 def fetch_with_retry(client, query, variables, max_retries=5):
@@ -169,12 +169,14 @@ def collect_all_mech_delivers(from_block: int, to_block: int, filename: str) ->
 
     mech_delivers = {}
     duplicated_requestIds = []
-    transport = RequestsHTTPTransport(url=THEGRAPH_ENDPOINT)
+    mech_subgraph_url = MECH_SUBGRAPH_URL.substitute(subgraph_api_key=SUBGRAPH_API_KEY)
+    transport = RequestsHTTPTransport(url=mech_subgraph_url)
     client = Client(transport=transport, fetch_schema_from_transport=True)
     to_block = (
         to_block + MECH_FROM_BLOCK_RANGE
     )  # there is a delay between deliver and request
     id_gt = ""
+    nr_errors = 0
     while True:
         variables = {
             "id_gt": id_gt,
@@ -182,9 +184,6 @@ def collect_all_mech_delivers(from_block: int, to_block: int, filename: str) ->
             "blockNumber_lte": str(to_block),  # str
         }
         try:
-            # response = client.execute(
-            #     gql(DELIVERS_QUERY_NO_FILTER), variable_values=variables
-            # )
             response = fetch_with_retry(client, DELIVERS_QUERY_NO_FILTER, variables)
             items = response.get("delivers", [])
 
@@ -198,8 +197,10 @@ def collect_all_mech_delivers(from_block: int, to_block: int, filename: str) ->
                     duplicated_requestIds.append(mech_deliver["requestId"])
                     # we will handle the duplicated later
         except Exception as e:
+            # counter for errors
+            nr_errors += 1
             print(f"Error while getting the response: {e}")
-            return None, None
+            # return None, None
 
         id_gt = items[-1]["id"]
         time.sleep(IPFS_POLL_INTERVAL)
@@ -210,7 +211,7 @@ def collect_all_mech_delivers(from_block: int, to_block: int, filename: str) ->
     print(f"Number of delivers = {len(mech_delivers)}")
     print(f"Number of duplicated request id = {len(duplicated_requestIds)}")
     save_json_file(mech_delivers, filename)
-    return mech_delivers, duplicated_requestIds
+    return mech_delivers, duplicated_requestIds, nr_errors
 
 
 def collect_missing_delivers(request_id: int, block_number: int) -> Dict[str, Any]:
@@ -219,7 +220,8 @@ def collect_missing_delivers(request_id: int, block_number: int) -> Dict[str, An
     )  # there is a delay between deliver and request
     print(f"Fetching all missing delivers from {block_number} to {to_block}")
     mech_delivers = {}
-    transport = RequestsHTTPTransport(url=THEGRAPH_ENDPOINT)
+    mech_subgraph_url = MECH_SUBGRAPH_URL.substitute(subgraph_api_key=SUBGRAPH_API_KEY)
+    transport = RequestsHTTPTransport(url=mech_subgraph_url)
     client = Client(transport=transport, fetch_schema_from_transport=True)
 
     variables = {
@@ -277,7 +279,7 @@ def populate_requests_ipfs_contents(
         updated_dict[k] = mech_request
         time.sleep(IPFS_POLL_INTERVAL)
 
-    return updated_dict
+    return updated_dict, wrong_response_count
 
 
 def populate_delivers_ipfs_contents(
@@ -285,6 +287,7 @@ def populate_delivers_ipfs_contents(
 ) -> dict:
     """Function to complete the delivers content info from ipfs"""
     updated_dict = {}
+    errors = 0
     for k in tqdm(
         keys_to_traverse,
         desc="Fetching IPFS contents for delivers",
@@ -316,6 +319,7 @@ def populate_delivers_ipfs_contents(
                 tqdm.write(f"Skipping {mech_request} because of JSONDecodeError")
                 continue
             except Exception:
+                errors += 1
                 tqdm.write(
                     f"Skipping {mech_request} because of error parsing the response"
                 )
@@ -323,7 +327,7 @@ def populate_delivers_ipfs_contents(
         updated_dict[k] = mech_request
         time.sleep(IPFS_POLL_INTERVAL)
 
-    return updated_dict
+    return updated_dict, errors
 
 
 def write_mech_events_to_file(
@@ -500,6 +504,7 @@ def get_ipfs_data(input_filename: str, output_filename: str):
     session = create_session()
     print("UPDATING IPFS CONTENTS OF REQUESTS")
     # requests
+    nr_errors = 0
     with ThreadPoolExecutor(max_workers=NUM_WORKERS) as executor:
         futures = []
         for i in range(0, len(mech_requests), GET_CONTENTS_BATCH_SIZE):
@@ -517,12 +522,15 @@ def get_ipfs_data(input_filename: str, output_filename: str):
             total=len(futures),
             desc=f"Fetching all ipfs contents from requests ",
         ):
-            partial_dict = future.result()
+            partial_dict, error_counter = future.result()
+            nr_errors += error_counter
             updated_mech_requests.update(partial_dict)
 
     save_json_file(updated_mech_requests, output_filename)
+    print(f"NUMBER OF MECH REQUEST IPFS ERRORS={nr_errors}")
 
     # delivers
+    nr_deliver_errors = 0
     print("UPDATING IPFS CONTENTS OF DELIVERS")
     total_keys_to_traverse = list(updated_mech_requests.keys())
     final_tools_content = {}
@@ -543,10 +551,12 @@ def get_ipfs_data(input_filename: str, output_filename: str):
             total=len(futures),
             desc=f"Fetching all ipfs contents from delivers ",
         ):
-            partial_dict = future.result()
+            partial_dict, error_counter = future.result()
+            nr_deliver_errors += error_counter
             final_tools_content.update(partial_dict)
 
     save_json_file(final_tools_content, output_filename)
+    print(f"NUMBER OF MECH DELIVERS IPFS ERRORS={nr_deliver_errors}")
 
 
 def only_delivers_loop():

scripts/nr_mech_calls.py

@@ -1,6 +1,15 @@
 import pandas as pd
 from utils import DATA_DIR, DEFAULT_MECH_FEE
 from tqdm import tqdm
+from datetime import datetime, timezone
+from typing import Dict, Any
+from collections import defaultdict
+from tools import IRRELEVANT_TOOLS
+import re
+
+
+def transform_to_datetime(x):
+    return datetime.fromtimestamp(int(x), tz=timezone.utc)
 
 
 def update_roi(row: pd.DataFrame) -> float:
@@ -12,6 +21,38 @@ def update_roi(row: pd.DataFrame) -> float:
     return new_value
 
 
+def get_mech_statistics(mech_requests: Dict[str, Any]) -> Dict[str, Dict[str, int]]:
+    """Outputs a table with Mech statistics"""
+
+    mech_statistics: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int))
+
+    for mech_request in mech_requests.values():
+        if (
+            "ipfs_contents" not in mech_request
+            or "tool" not in mech_request["ipfs_contents"]
+            or "prompt" not in mech_request["ipfs_contents"]
+        ):
+            continue
+
+        if mech_request["ipfs_contents"]["tool"] in IRRELEVANT_TOOLS:
+            continue
+
+        prompt = mech_request["ipfs_contents"]["prompt"]
+        prompt = prompt.replace("\n", " ")
+        prompt = prompt.strip()
+        prompt = re.sub(r"\s+", " ", prompt)
+        prompt_match = re.search(r"\"(.*)\"", prompt)
+        if prompt_match:
+            question = prompt_match.group(1)
+        else:
+            question = prompt
+
+        mech_statistics[question]["count"] += 1
+        mech_statistics[question]["fees"] += mech_request["fee"]
+
+    return mech_statistics
+
+
 def create_unknown_traders_df(trades_df: pd.DataFrame) -> pd.DataFrame:
     """filter trades coming from non-Olas traders that are placing no mech calls"""
     no_mech_calls_mask = (trades_df["staking"] == "non_Olas") & (
@@ -66,6 +107,163 @@ def update_trade_nr_mech_calls(non_agents: bool = False):
     # summary_df.to_parquet(DATA_DIR / "summary_profitability.parquet", index=False)
 
 
+def get_daily_mech_calls_estimation(
+    daily_trades: pd.DataFrame, daily_tools: pd.DataFrame
+) -> list:
+    # for each market
+    daily_markets = daily_trades.title.unique()
+    trader = daily_trades.iloc[0].trader_address
+    day = daily_trades.iloc[0].creation_date
+    estimations = []
+    for market in daily_markets:
+        estimation_dict = {}
+        estimation_dict["trader_address"] = trader
+        estimation_dict["trading_day"] = day
+        # tools usage of this market
+        market_requests = daily_tools.loc[daily_tools["title"] == market]
+        # trades done on this market
+        market_trades = daily_trades[daily_trades["title"] == market]
+        mech_calls_estimation = 0
+        total_trades = len(market_trades)
+        total_requests = 0
+        if len(market_requests) > 0:
+            total_requests = len(market_requests)
+            mech_calls_estimation = total_requests / total_trades
+        estimation_dict["total_trades"] = total_trades
+        estimation_dict["total_mech_requests"] = total_requests
+        estimation_dict["market"] = market
+        estimation_dict["mech_calls_per_trade"] = mech_calls_estimation
+        estimations.append(estimation_dict)
+    return estimations
+
+
+def compute_daily_mech_calls(
+    fpmmTrades: pd.DataFrame, tools: pd.DataFrame
+) -> pd.DataFrame:
+    """Function to compute the daily mech calls at the trader and market level"""
+    nr_traders = len(fpmmTrades["trader_address"].unique())
+    fpmmTrades["creation_timestamp"] = pd.to_datetime(fpmmTrades["creationTimestamp"])
+    fpmmTrades["creation_date"] = fpmmTrades["creation_timestamp"].dt.date
+    trades_df = trades_df.sort_values(by="creation_timestamp", ascending=True)
+    tools["request_time"] = pd.to_datetime(tools["request_time"])
+    tools["request_date"] = tools["request_time"].dt.date
+    tools = tools.sort_values(by="request_time", ascending=True)
+    all_mech_calls = []
+    for trader in tqdm(
+        fpmmTrades["trader_address"].unique(),
+        total=nr_traders,
+        desc="creating mech calls estimation based on timestamps",
+    ):
+        # compute the mech calls estimations for each trader
+        all_trades = fpmmTrades[fpmmTrades["trader_address"] == trader]
+        all_tools = tools[tools["trader_address"] == trader]
+        trading_days = all_trades.creation_date.unique()
+        for trading_day in trading_days:
+            daily_trades = all_trades.loc[all_trades["creation_date"] == trading_day]
+            daily_tools = all_tools.loc[all_tools["request_date"] == trading_day]
+            trader_entry = {}
+            trader_entry["trader_address"] = trader
+            trader_entry["total_trades"] = len(daily_trades)
+            trader_entry["trading_day"] = trading_day
+            trader_entry["total_mech_calls"] = len(daily_tools)
+            all_mech_calls.append(trader_entry)
+    return pd.DataFrame.from_dict(all_mech_calls, orient="columns")
+
+
+def compute_mech_call_estimations(
+    fpmmTrades: pd.DataFrame, tools: pd.DataFrame
+) -> pd.DataFrame:
+    """Function to compute the estimated mech calls needed per trade at the trader and market level"""
+    nr_traders = len(fpmmTrades["trader_address"].unique())
+    fpmmTrades["creation_timestamp"] = pd.to_datetime(fpmmTrades["creationTimestamp"])
+    fpmmTrades["creation_date"] = fpmmTrades["creation_timestamp"].dt.date
+    tools["request_time"] = pd.to_datetime(tools["request_time"])
+    tools["request_date"] = tools["request_time"].dt.date
+    all_estimations = []
+    for trader in tqdm(
+        fpmmTrades["trader_address"].unique(),
+        total=nr_traders,
+        desc="creating mech calls estimation dataframe",
+    ):
+        # compute the mech calls estimations for each trader
+        all_trades = fpmmTrades[fpmmTrades["trader_address"] == trader]
+        all_tools = tools[tools["trader_address"] == trader]
+        trading_days = all_trades.creation_date.unique()
+        for trading_day in trading_days:
+            daily_trades = all_trades.loc[all_trades["creation_date"] == trading_day]
+            daily_tools = all_tools.loc[all_tools["request_date"] == trading_day]
+            daily_estimations = get_daily_mech_calls_estimation(
+                daily_trades=daily_trades, daily_tools=daily_tools
+            )
+            all_estimations.extend(daily_estimations)
+    return pd.DataFrame.from_dict(all_estimations, orient="columns")
+
+
+def compute_timestamp_mech_calls(
+    all_trades: pd.DataFrame, all_tools: pd.DataFrame
+) -> list:
+    """Function to compute the mech calls based on timestamps but without repeating mech calls"""
+    mech_calls_contents = []
+    request_timestamps_used = {}
+    # intialize the dict with all markets
+    all_markets = all_trades.title.unique()
+    for market in all_markets:
+        request_timestamps_used[market] = []
+
+    for i, trade in all_trades.iterrows():
+        trader = trade["trader_address"]
+        trade_id = trade["id"]
+        market = trade["title"]
+        trade_ts = trade["creation_timestamp"]
+        market_requests = all_tools.loc[
+            (all_tools["trader_address"] == trader) & (all_tools["title"] == market)
+        ]
+        # traverse market requests
+        total_mech_calls = 0
+        for mech_request in market_requests:
+            # check timestamp (before the trade)
+            request_ts = mech_request.request_time
+            if request_ts < trade_ts:
+                # check the timestamp has not been used in a previous trade
+                used_timestamps = request_timestamps_used[market]
+                if request_ts not in used_timestamps:
+                    request_timestamps_used[market].append(request_ts)
+                    total_mech_calls += 1
+        # create enty for the dataframe
+        mech_call_entry = {}
+        mech_call_entry["trader_address"] = trader
+        mech_call_entry["market"] = market
+        mech_call_entry["trade_id"] = trade_id
+        mech_call_entry["total_mech_calls"] = total_mech_calls
+        mech_calls_contents.append(mech_call_entry)
+    return mech_calls_contents
+
+
+def compute_mech_calls_based_on_timestamps(
+    fpmmTrades: pd.DataFrame, tools: pd.DataFrame
+) -> pd.DataFrame:
+    """Function to compute the mech calls needed per trade at the trader and market level using timestamps"""
+    nr_traders = len(fpmmTrades["trader_address"].unique())
+    fpmmTrades["creation_timestamp"] = pd.to_datetime(fpmmTrades["creationTimestamp"])
+    fpmmTrades["creation_date"] = fpmmTrades["creation_timestamp"].dt.date
+    trades_df = trades_df.sort_values(by="creation_timestamp", ascending=True)
+    tools["request_time"] = pd.to_datetime(tools["request_time"])
+    tools["request_date"] = tools["request_time"].dt.date
+    tools = tools.sort_values(by="request_time", ascending=True)
+    all_mech_calls = []
+    for trader in tqdm(
+        fpmmTrades["trader_address"].unique(),
+        total=nr_traders,
+        desc="creating mech calls estimation based on timestamps",
+    ):
+        # compute the mech calls estimations for each trader
+        all_trades = fpmmTrades[fpmmTrades["trader_address"] == trader]
+        all_tools = tools[tools["trader_address"] == trader]
+        trader_mech_calls = compute_timestamp_mech_calls(all_trades, all_tools)
+        all_mech_calls.extend(trader_mech_calls)
+    return pd.DataFrame.from_dict(all_mech_calls, orient="columns")
+
+
 if __name__ == "__main__":
     # update_trade_nr_mech_calls(non_agents=True)
     trades_df = pd.read_parquet(DATA_DIR / "all_trades_profitability.parquet")

scripts/profitability.py

@@ -40,7 +40,11 @@ from utils import (
     DEFAULT_MECH_FEE,
 )
 from staking import label_trades_by_staking
-from nr_mech_calls import create_unknown_traders_df
+from nr_mech_calls import (
+    create_unknown_traders_df,
+    transform_to_datetime,
+    compute_mech_calls_based_on_timestamps,
+)
 
 DUST_THRESHOLD = 10000000000000
 INVALID_ANSWER = -1
@@ -215,12 +219,14 @@ def analyse_trader(
     trader_address: str,
     fpmmTrades: pd.DataFrame,
     tools: pd.DataFrame,
+    trader_estimated_mech_calls: pd.DataFrame,
     daily_info: bool = False,
 ) -> pd.DataFrame:
     """Analyse a trader's trades"""
+    fpmmTrades["creation_timestamp"] = pd.to_datetime(fpmmTrades["creationTimestamp"])
+    fpmmTrades["creation_date"] = fpmmTrades["creation_timestamp"].dt.date
     # Filter trades and tools for the given trader
     trades = fpmmTrades[fpmmTrades["trader_address"] == trader_address]
-    tools_usage = tools[tools["trader_address"] == trader_address]
 
     # Prepare the DataFrame
     trades_df = pd.DataFrame(columns=ALL_TRADES_STATS_DF_COLS)
@@ -238,14 +244,12 @@ def analyse_trader(
     for i, trade in tqdm(trades.iterrows(), total=len(trades), desc="Analysing trades"):
         try:
             market_answer = trade["fpmm.currentAnswer"]
+            trading_day = trade["creation_date"]
+            trade_id = trade["id"]
             if not daily_info and not market_answer:
                 print(f"Skipping trade {i} because currentAnswer is NaN")
                 continue
             # Parsing and computing shared values
-
-            creation_timestamp_utc = datetime.datetime.fromtimestamp(
-                int(trade["creationTimestamp"]), tz=datetime.timezone.utc
-            )
             collateral_amount = wei_to_unit(float(trade["collateralAmount"]))
             fee_amount = wei_to_unit(float(trade["feeAmount"]))
             outcome_tokens_traded = wei_to_unit(float(trade["outcomeTokensTraded"]))
@@ -280,25 +284,23 @@ def analyse_trader(
                 earnings = outcome_tokens_traded
                 winner_trade = True
 
-            # Compute mech calls
-            if len(tools_usage) == 0:
-                print("No tools usage information")
-                num_mech_calls = 0
+            # Compute mech calls using the title, and trade id
+            if daily_info:
+                total_mech_calls = trader_estimated_mech_calls.loc[
+                    (trader_estimated_mech_calls["trading_day"] == trading_day),
+                    "total_mech_calls",
+                ].iloc[0]
             else:
-                try:
-                    num_mech_calls = (
-                        tools_usage["prompt_request"]
-                        .apply(lambda x: trade["title"] in x)
-                        .sum()
-                    )
-                except Exception:
-                    print(f"Error while getting the number of mech calls")
-                    num_mech_calls = 2  # Average value
+                total_mech_calls = trader_estimated_mech_calls.loc[
+                    (trader_estimated_mech_calls["market"] == trade["title"])
+                    & (trader_estimated_mech_calls["trade_id"] == trade_id),
+                    "mech_calls_per_trade",
+                ].iloc[0]
 
             net_earnings = (
                 earnings
                 - fee_amount
-                - (num_mech_calls * DEFAULT_MECH_FEE)
+                - (total_mech_calls * DEFAULT_MECH_FEE)
                 - collateral_amount
             )
 
@@ -308,7 +310,7 @@ def analyse_trader(
                 "market_creator": market_creator,
                 "trade_id": trade["id"],
                 "market_status": market_status.name,
-                "creation_timestamp": creation_timestamp_utc,
+                "creation_timestamp": trade["creationTimestamp"],
                 "title": trade["title"],
                 "collateral_amount": collateral_amount,
                 "outcome_index": trade["outcomeIndex"],
@@ -320,11 +322,13 @@ def analyse_trader(
                 "earnings": earnings,
                 "redeemed": redemption,
                 "redeemed_amount": earnings if redemption else 0,
-                "num_mech_calls": num_mech_calls,
-                "mech_fee_amount": num_mech_calls * DEFAULT_MECH_FEE,
+                "num_mech_calls": total_mech_calls,
+                "mech_fee_amount": total_mech_calls * DEFAULT_MECH_FEE,
                 "net_earnings": net_earnings,
                 "roi": net_earnings
-                / (collateral_amount + fee_amount + num_mech_calls * DEFAULT_MECH_FEE),
+                / (
+                    collateral_amount + fee_amount + total_mech_calls * DEFAULT_MECH_FEE
+                ),
             }
 
         except Exception as e:
@@ -336,16 +340,27 @@ def analyse_trader(
 
 
 def analyse_all_traders(
-    trades: pd.DataFrame, tools: pd.DataFrame, daily_info: bool = False
+    trades: pd.DataFrame,
+    tools: pd.DataFrame,
+    estimated_mech_calls: pd.DataFrame,
+    daily_info: bool = False,
 ) -> pd.DataFrame:
     """Analyse all creators."""
+
     all_traders = []
     for trader in tqdm(
         trades["trader_address"].unique(),
         total=len(trades["trader_address"].unique()),
         desc="Analysing creators",
     ):
-        all_traders.append(analyse_trader(trader, trades, tools, daily_info))
+        trader_estimated_mech_calls = estimated_mech_calls.loc[
+            estimated_mech_calls["trader_address"] == trader
+        ]
+        all_traders.append(
+            analyse_trader(
+                trader, trades, tools, trader_estimated_mech_calls, daily_info
+            )
+        )
 
     # concat all creators
     all_creators_df = pd.concat(all_traders)
@@ -414,8 +429,16 @@ def run_profitability_analysis(
         update_tools_parquet(rpc, tools_filename)
     tools = pd.read_parquet(DATA_DIR / "tools.parquet")
 
+    fpmmTrades["creationTimestamp"] = fpmmTrades["creationTimestamp"].apply(
+        lambda x: transform_to_datetime(x)
+    )
+    print("Computing the estimated mech calls dataset")
+    trade_mech_calls = compute_mech_calls_based_on_timestamps(
+        fpmmTrades=fpmmTrades, tools=tools
+    )
+    print(trade_mech_calls.total_mech_calls.describe())
     print("Analysing trades...")
-    all_trades_df = analyse_all_traders(fpmmTrades, tools)
+    all_trades_df = analyse_all_traders(fpmmTrades, tools, trade_mech_calls)
 
     # # merge previous files if requested
     if merge:

scripts/pull_data.py

@@ -3,7 +3,7 @@ from datetime import datetime
 import pandas as pd
 from markets import etl as mkt_etl, DEFAULT_FILENAME as MARKETS_FILENAME, fpmmTrades_etl
 from tools import DEFAULT_FILENAME as TOOLS_FILENAME, generate_tools_file
-from profitability import run_profitability_analysis, DEFAULT_60_DAYS_AGO_TIMESTAMP
+from profitability import run_profitability_analysis
 from utils import (
     get_question,
     current_answer,
@@ -13,7 +13,6 @@ from utils import (
     HIST_DIR,
 )
 from get_mech_info import (
-    get_mech_events_last_60_days,
     get_mech_events_since_last_run,
     update_json_files,
 )
@@ -57,7 +56,7 @@ def save_historical_data():
         filename = f"tools_{timestamp}.parquet"
         tools.to_parquet(HIST_DIR / filename, index=False)
         # save into cloud storage
-        # load_historical_file(filename)
+        load_historical_file(filename)
     except Exception as e:
         print(f"Error saving tools file in the historical folder {e}")
 
@@ -66,7 +65,7 @@ def save_historical_data():
         filename = f"all_trades_profitability_{timestamp}.parquet"
         all_trades.to_parquet(HIST_DIR / filename, index=False)
         # save into cloud storage
-        # load_historical_file(filename)
+        load_historical_file(filename)
 
     except Exception as e:
         print(
@@ -80,7 +79,7 @@ def only_new_weekly_analysis():
     rpc = RPC
     # Run markets ETL
     logging.info("Running markets ETL")
-    # mkt_etl(MARKETS_FILENAME)
+    mkt_etl(MARKETS_FILENAME)
     logging.info("Markets ETL completed")
 
     # Mech events ETL
@@ -128,7 +127,7 @@ def only_new_weekly_analysis():
 
     save_historical_data()
 
-    clean_old_data_from_parquet_files("2024-10-14")
+    clean_old_data_from_parquet_files("2024-10-17")
 
     compute_tools_accuracy()
 

scripts/utils.py

@@ -9,6 +9,7 @@ import re
 from dataclasses import dataclass
 from pathlib import Path
 from enum import Enum
+from string import Template
 from json.decoder import JSONDecodeError
 
 DEFAULT_MECH_FEE = 0.01
@@ -48,6 +49,21 @@ INC_TOOLS = [
     "prediction-request-reasoning-claude",
     "superforcaster",
 ]
+SUBGRAPH_URL = Template(
+    """https://gateway-arbitrum.network.thegraph.com/api/${subgraph_api_key}/subgraphs/id/7s9rGBffUTL8kDZuxvvpuc46v44iuDarbrADBFw5uVp2"""
+)
+OMEN_SUBGRAPH_URL = Template(
+    """https://gateway-arbitrum.network.thegraph.com/api/${subgraph_api_key}/subgraphs/id/9fUVQpFwzpdWS9bq5WkAnmKbNNcoBwatMR4yZq81pbbz"""
+)
+NETWORK_SUBGRAPH_URL = Template(
+    """https://gateway-arbitrum.network.thegraph.com/api/${subgraph_api_key}/subgraphs/id/FxV6YUix58SpYmLBwc9gEHkwjfkqwe1X5FJQjn8nKPyA"""
+)
+# THEGRAPH_ENDPOINT = (
+#     "https://api.studio.thegraph.com/query/78829/mech-predict/version/latest"
+# )
+MECH_SUBGRAPH_URL = Template(
+    """https://gateway.thegraph.com/api/${subgraph_api_key}/subgraphs/id/4YGoX3iXUni1NBhWJS5xyKcntrAzssfytJK7PQxxQk5g"""
+)
 
 SUBGRAPH_API_KEY = os.environ.get("SUBGRAPH_API_KEY", None)
 RPC = os.environ.get("RPC", None)

scripts/web3_utils.py

@@ -12,7 +12,7 @@ from tqdm import tqdm
 from web3 import Web3
 from typing import Any, Optional
 from web3.types import BlockParams
-from utils import JSON_DATA_DIR, DATA_DIR, SUBGRAPH_API_KEY, to_content
+from utils import JSON_DATA_DIR, DATA_DIR, SUBGRAPH_API_KEY, to_content, SUBGRAPH_URL
 from queries import conditional_tokens_gc_user_query, omen_xdai_trades_query
 import pandas as pd
 
@@ -30,6 +30,9 @@ N_RPC_RETRIES = 100
 RPC_POLL_INTERVAL = 0.05
 # IPFS_POLL_INTERVAL = 0.05  # low speed
 IPFS_POLL_INTERVAL = 0.2  # high speed
+OMEN_SUBGRAPH_URL = Template(
+    """https://gateway-arbitrum.network.thegraph.com/api/${subgraph_api_key}/subgraphs/id/9fUVQpFwzpdWS9bq5WkAnmKbNNcoBwatMR4yZq81pbbz"""
+)
 
 headers = {
     "Accept": "application/json, multipart/mixed",
@@ -156,9 +159,7 @@ def updating_timestamps(rpc: str, tools_filename: str):
 
 def query_conditional_tokens_gc_subgraph(creator: str) -> dict[str, Any]:
     """Query the subgraph."""
-    SUBGRAPH_URL = Template(
-        """https://gateway-arbitrum.network.thegraph.com/api/${subgraph_api_key}/subgraphs/id/7s9rGBffUTL8kDZuxvvpuc46v44iuDarbrADBFw5uVp2"""
-    )
+
     subgraph = SUBGRAPH_URL.substitute(subgraph_api_key=SUBGRAPH_API_KEY)
     all_results: dict[str, Any] = {"data": {"user": {"userPositions": []}}}
     userPositions_id_gt = ""
@@ -200,9 +201,7 @@ def query_omen_xdai_subgraph(
     fpmm_to_timestamp: float,
 ) -> dict[str, Any]:
     """Query the subgraph."""
-    OMEN_SUBGRAPH_URL = Template(
-        """https://gateway-arbitrum.network.thegraph.com/api/${subgraph_api_key}/subgraphs/id/9fUVQpFwzpdWS9bq5WkAnmKbNNcoBwatMR4yZq81pbbz"""
-    )
+
     omen_subgraph = OMEN_SUBGRAPH_URL.substitute(subgraph_api_key=SUBGRAPH_API_KEY)
     print(f"omen_subgraph = {omen_subgraph}")
     grouped_results = defaultdict(list)

tabs/metrics.py

@@ -187,13 +187,43 @@ def plot_trade_metrics(
     )
 
 
-def get_trade_metrics_text() -> gr.Markdown:
-    metric_text = """ 
-        ## Description of the graph
-        These metrics are computed weekly. The statistical measures are:
-        * min, max, 25th(q1), 50th(median) and 75th(q2) percentiles
-        * the upper and lower fences to delimit possible outliers
-        * the average values as the dotted lines
-        """
-
+def get_trade_metrics_text(trader_type: str = None) -> gr.Markdown:
+    if trader_type is None:
+        metric_text = """ 
+            ## Description of the graph
+            These metrics are computed weekly. The statistical measures are:
+            * min, max, 25th(q1), 50th(median) and 75th(q2) percentiles
+            * the upper and lower fences to delimit possible outliers
+            * the average values as the dotted lines
+            """
+    elif trader_type == "Olas":
+        metric_text = """ 
+            ## Definition of Olas trader
+            Agents using Mech, with a service ID and the corresponding safe in the registry
+            ## Description of the graph
+            These metrics are computed weekly. The statistical measures are:
+            * min, max, 25th(q1), 50th(median) and 75th(q2) percentiles
+            * the upper and lower fences to delimit possible outliers
+            * the average values as the dotted lines
+            """
+    elif trader_type == "non_Olas":
+        metric_text = """ 
+            ## Definition of non-Olas trader
+            Agents using Mech, with no service ID
+            ## Description of the graph
+            These metrics are computed weekly. The statistical measures are:
+            * min, max, 25th(q1), 50th(median) and 75th(q2) percentiles
+            * the upper and lower fences to delimit possible outliers
+            * the average values as the dotted lines
+            """
+    else:  # Unclassified
+        metric_text = """ 
+            ## Definition of unclassified trader
+            Agents (safe/EOAs) not using Mechs
+            ## Description of the graph
+            These metrics are computed weekly. The statistical measures are:
+            * min, max, 25th(q1), 50th(median) and 75th(q2) percentiles
+            * the upper and lower fences to delimit possible outliers
+            * the average values as the dotted lines
+            """
     return gr.Markdown(metric_text)