updating all files except the tools big file

app.py

@@ -201,20 +201,20 @@ with demo:
             with gr.Row():
                 with gr.Column(scale=1):
                     gr.Markdown(
-                        "# Weekly percentage of winning for  🤖 Agent based trades"
+                        "# Weekly percentage of winning for trades based on 🌊 Olas traders"
                     )
-                    agent_winning_trades = (
+                    olas_winning_trades = (
                         integrated_plot_winning_trades_per_market_by_week_v2(
-                            trades_df=trades_df, trader_filter="agent"
+                            trades_df=trades_df, trader_filter="Olas"
                         )
                     )
                 with gr.Column(scale=1):
                     gr.Markdown(
-                        "# Weekly percentage of winning for Non-agent based trades"
+                        "# Weekly percentage of winning for trades based on non-Olas traders"
                     )
-                    non_agent_winning_trades = (
+                    non_Olas_winning_trades = (
                         integrated_plot_winning_trades_per_market_by_week_v2(
-                            trades_df=trades_df, trader_filter="non_agent"
+                            trades_df=trades_df, trader_filter="non_Olas"
                         )
                     )
 
@@ -251,7 +251,9 @@ with demo:
 
             # Agentic traders graph
             with gr.Row():
-                gr.Markdown("# Weekly trading metrics for trades coming from Agents 🤖")
+                gr.Markdown(
+                    "# Weekly trading metrics for trades coming from 🌊 Olas traders"
+                )
             with gr.Row():
                 trade_a_details_selector = gr.Dropdown(
                     label="Select a trade metric",
@@ -273,7 +275,7 @@ with demo:
                 new_a_plot = plot_trade_metrics(
                     metric_name=trade_detail,
                     trades_df=trades_df,
-                    trader_filter="agent",
+                    trader_filter="Olas",
                 )
                 return new_a_plot
 
@@ -283,10 +285,10 @@ with demo:
                 outputs=[a_trade_details_plot],
             )
 
-            # Non-agentic traders graph
+            # Non-Olasic traders graph
             with gr.Row():
                 gr.Markdown(
-                    "# Weekly trading metrics for trades coming from Non-agents"
+                    "# Weekly trading metrics for trades coming from Non-Olas traders"
                 )
             with gr.Row():
                 trade_na_details_selector = gr.Dropdown(
@@ -300,7 +302,7 @@ with demo:
                     na_trade_details_plot = plot_trade_metrics(
                         metric_name=default_trade_metric,
                         trades_df=trades_df,
-                        trader_filter="non_agent",
+                        trader_filter="non_Olas",
                     )
                 with gr.Column(scale=1):
                     trade_details_text = get_trade_metrics_text()
@@ -309,7 +311,7 @@ with demo:
                 new_a_plot = plot_trade_metrics(
                     metric_name=trade_detail,
                     trades_df=trades_df,
-                    trader_filter="non_agent",
+                    trader_filter="non_Olas",
                 )
                 return new_a_plot
 
@@ -323,13 +325,13 @@ with demo:
             with gr.Row():
                 gr.Markdown("# Trades conducted at the Pearl markets")
             with gr.Row():
-                staking_trades_by_week = plot_staking_trades_per_market_by_week(
+                staking_pearl_trades_by_week = plot_staking_trades_per_market_by_week(
                     trades_df=trades_df, market_creator="pearl"
                 )
             with gr.Row():
                 gr.Markdown("# Trades conducted at the Quickstart markets")
             with gr.Row():
-                staking_trades_by_week = plot_staking_trades_per_market_by_week(
+                staking_qs_trades_by_week = plot_staking_trades_per_market_by_week(
                     trades_df=trades_df, market_creator="quickstart"
                 )
             with gr.Row():

data/all_trades_profitability.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0a2f046c08d9559564088d23ce788bf06b1d7e64eb1851659cb8e89bd4759e95
-size 3656355
+oid sha256:9ff7a9001dceeac25cf151e9b8cff55beafe610387c42e093d004e2712206e6b
+size 3884891

data/daily_info.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9b9c7c71ea81900aac2a840315940801489e15aa383c46467b30b02f7e3baea2
-size 461471
+oid sha256:d6d36bd873a5fe9564f556eda0210078a0e9c3d0dd81fec2158cf029ed462573
+size 911119

data/invalid_trades.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f1455a77d83f6a3a4d08f4cd7ab0a641f405f5344e6ef461aa5907c16727c46b
-size 108711
+oid sha256:88697b4baf7652f32c3413f1fc168f534f2472281761fa4e5208751f1a0bae56
+size 123705

data/service_map.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0e7cf0de5a9f340c54dc0add3310bdc20ac25a0d71f45a2d29ad621d4944b153
-size 90766
+oid sha256:574e816f13aee41f153346e0590bbaeb5115578faa6aa9129dbda8b49b4d7fd2
+size 90733

data/summary_profitability.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:93cbff2dc14663aed525b0d93a7bc1405819a4048418805af2b9ed9232b97ac8
-size 107799
+oid sha256:20a70aff0b89a48381a0cf73ffb65ae9a41002b81bec1dd1ded9e454b86e9245
+size 112166

data/tools_accuracy.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8cc8bc62d7496b96eb3ac81373628b35ec2c85ba9b50ad91f97c4bed59038a04
-size 1342
+oid sha256:5aa2346a2dba117d4285bcc997eae04b935b32d1290226d74a270c25db358fd0
+size 1341

scripts/profitability.py

@@ -32,12 +32,7 @@ from get_mech_info import (
     update_tools_parquet,
     update_all_trades_parquet,
 )
-from utils import (
-    wei_to_unit,
-    convert_hex_to_int,
-    JSON_DATA_DIR,
-    DATA_DIR,
-)
+from utils import wei_to_unit, convert_hex_to_int, JSON_DATA_DIR, DATA_DIR, TMP_DIR
 from staking import label_trades_by_staking
 
 DUST_THRESHOLD = 10000000000000
@@ -426,9 +421,16 @@ def run_profitability_analysis(
         all_trades_df = update_all_trades_parquet(all_trades_df)
 
     # debugging purposes
-    all_trades_df.to_parquet(JSON_DATA_DIR / "all_trades_df.parquet")
+    all_trades_df.to_parquet(JSON_DATA_DIR / "all_trades_df.parquet", index=False)
+
+    # filter trades coming from non-Olas traders that are placing no mech calls
+    no_mech_calls_mask = (all_trades_df["staking"] == "non_Olas") & (
+        all_trades_df.loc["num_mech_calls"] == 0
+    )
+    no_mech_calls_df = all_trades_df.loc[no_mech_calls_mask]
+    no_mech_calls_df.to_parquet(TMP_DIR / "no_mech_calls_trades.parquet", index=False)
+    all_trades_df = all_trades_df.loc[~no_mech_calls_mask]
 
-    # all_trades_df = pd.read_parquet(JSON_DATA_DIR / "all_trades_df.parquet")
     # filter invalid markets. Condition: "is_invalid" is True
     invalid_trades = all_trades_df.loc[all_trades_df["is_invalid"] == True]
     if len(invalid_trades) == 0:

scripts/pull_data.py

@@ -80,7 +80,7 @@ def only_new_weekly_analysis():
     rpc = RPC
     # Run markets ETL
     logging.info("Running markets ETL")
-    mkt_etl(MARKETS_FILENAME)
+    # mkt_etl(MARKETS_FILENAME)
     logging.info("Markets ETL completed")
 
     # Mech events ETL

scripts/staking.py

@@ -168,7 +168,7 @@ def get_trader_address_staking(trader_address: str, service_map: dict) -> str:
             break
 
     if found_key == -1:
-        return "non_agent"
+        return "non_Olas"
     owner = service_map[found_key]["owner_address"]
     return check_owner_staking_contract(owner_address=owner)
 
@@ -182,6 +182,7 @@ def label_trades_by_staking(trades_df: pd.DataFrame, start: int = None) -> pd.Da
         last_key = max(keys)
     else:
         last_key = start
+    print(f"last service key = {last_key}")
     update_service_map(start=last_key)
     all_traders = trades_df.trader_address.unique()
     trades_df["staking"] = ""
@@ -199,6 +200,9 @@ def label_trades_by_staking(trades_df: pd.DataFrame, start: int = None) -> pd.Da
 if __name__ == "__main__":
     # create_service_map()
     trades_df = pd.read_parquet(DATA_DIR / "all_trades_profitability.parquet")
-    label_trades_by_staking(trades_df=trades_df, start=20)
+    print("before labeling")
+    print(trades_df.staking.value_counts())
+    label_trades_by_staking(trades_df=trades_df, start=8)
+    print("after labeling")
     print(trades_df.staking.value_counts())
     trades_df.to_parquet(DATA_DIR / "all_trades_profitability.parquet", index=False)

scripts/web3_utils.py

@@ -137,8 +137,16 @@ def updating_timestamps(rpc: str, tools_filename: str):
     )
     t_map.update(new_timestamps)
 
+    # filtering old timestamps
+    cutoff_date = datetime(2024, 9, 9)
+    filtered_map = {
+        k: v
+        for k, v in t_map.items()
+        if datetime.strptime(v, "%Y-%m-%d %H:%M:%S") < cutoff_date
+    }
+
     with open(DATA_DIR / "t_map.pkl", "wb") as f:
-        pickle.dump(t_map, f)
+        pickle.dump(filtered_map, f)
 
     # clean and release all memory
     del tools

tabs/metrics.py

@@ -145,14 +145,14 @@ def plot_trade_metrics(
         yaxis_title = "Gross profit per trade (xDAI)"
 
     color_discrete = ["purple", "darkgoldenrod", "darkgreen"]
-    if trader_filter == "agent":
+    if trader_filter == "Olas":
         trades_filtered = get_boxplot_metrics(
-            column_name, trades_df.loc[trades_df["staking"] != "non_agent"]
+            column_name, trades_df.loc[trades_df["staking"] != "non_Olas"]
         )
         color_discrete = ["darkviolet", "goldenrod", "green"]
-    elif trader_filter == "non_agent":
+    elif trader_filter == "non_Olas":
         trades_filtered = get_boxplot_metrics(
-            column_name, trades_df.loc[trades_df["staking"] == "non_agent"]
+            column_name, trades_df.loc[trades_df["staking"] == "non_Olas"]
         )
     else:
         trades_filtered = get_boxplot_metrics(column_name, trades_df)

tabs/staking.py

@@ -39,21 +39,32 @@ def plot_staking_trades_per_market_by_week(
     all_filtered_trades = all_filtered_trades.loc[
         all_filtered_trades["market_creator"] == market_creator
     ]
-
+    print(f"Checking values for market creator={market_creator}")
+    print(all_filtered_trades.staking.value_counts())
     if market_creator != "all":
+        if market_creator == "pearl":
+            # remove the staking data from quickstart
+            all_filtered_trades = all_filtered_trades.loc[
+                all_filtered_trades["staking"] != "quickstart"
+            ]
+        else:
+            # remove the staking data from pearl
+            all_filtered_trades = all_filtered_trades.loc[
+                all_filtered_trades["staking"] != "pearl"
+            ]
         all_filtered_trades["staking"] = all_filtered_trades["staking"].replace(
-            {market_creator: "staking_traders", "non_agent": "non_agent_traders"}
+            {market_creator: "staking_traders", "non_Olas": "non_Olas_traders"}
         )
         colour_sequence = ["gray", market_colour, "black"]
         categories_sorted = {
-            "staking": ["non_staking_traders", "staking_traders", "non_agent_traders"]
+            "staking": ["non_staking_traders", "staking_traders", "non_Olas_traders"]
         }
     else:
         all_filtered_trades["staking"] = all_filtered_trades["staking"].replace(
             {
                 "pearl": "staking_pearl_traders",
                 "quickstart": "staking_quickstart_traders",
-                "non_agent": "non_agent_traders",
+                "non_Olas": "non_Olas_traders",
             }
         )
         colour_sequence = ["gray", "darkviolet", "goldenrod", "black"]
@@ -62,7 +73,7 @@ def plot_staking_trades_per_market_by_week(
                 "non_staking_traders",
                 "staking_pearl_traders",
                 "staking_quickstart_traders",
-                "non_agent_traders",
+                "non_Olas_traders",
             ]
         }
     trades = get_overall_by_staking_traders(all_filtered_trades)

tabs/trades.py

@@ -165,7 +165,7 @@ def integrated_plot_trades_per_market_by_week_v2(trades_df: pd.DataFrame) -> gr.
     )
     # Create binary staking category
     all_filtered_trades["staking_type"] = all_filtered_trades["staking"].apply(
-        lambda x: "non_agent" if x == "non_agent" else "agent"
+        lambda x: "non_Olas" if x == "non_Olas" else "Olas"
     )
 
     # Group by week, market_creator and staking_type
@@ -194,34 +194,34 @@ def integrated_plot_trades_per_market_by_week_v2(trades_df: pd.DataFrame) -> gr.
         "all": "darkgreen",
     }
 
-    # Process both agent and non-agent traces for each market together
+    # Process both Olas and non-Olas traces for each market together
     for market in ["pearl", "quickstart", "all"]:
         market_data = trades[trades["market_creator"] == market]
 
-        # First add 'agent' trace
-        agent_data = market_data[market_data["staking_type"] == "agent"]
-        agent_trace = go.Bar(
-            x=agent_data["month_year_week"],
-            y=agent_data["trades"],
-            name=f"{market}-agent",
+        # First add 'Olas' trace
+        olas_data = market_data[market_data["staking_type"] == "Olas"]
+        olas_trace = go.Bar(
+            x=olas_data["month_year_week"],
+            y=olas_data["trades"],
+            name=f"{market}-Olas",
             marker_color=market_colors[market],
             offsetgroup=market,  # Keep the market grouping
             showlegend=True,
         )
 
-        # Then add 'non_agent' trace with base set to agent values
-        non_agent_data = market_data[market_data["staking_type"] == "non_agent"]
-        non_agent_trace = go.Bar(
-            x=non_agent_data["month_year_week"],
-            y=non_agent_data["trades"],
-            name=f"{market}-non_agent",
+        # Then add 'non_Olas' trace with base set to olas values
+        non_Olas_data = market_data[market_data["staking_type"] == "non_Olas"]
+        non_Olas_trace = go.Bar(
+            x=non_Olas_data["month_year_week"],
+            y=non_Olas_data["trades"],
+            name=f"{market}-non_Olas",
             marker_color=market_darker_colors[market],
             offsetgroup=market,  # Keep the market grouping
-            base=agent_data["trades"],  # Stack on top of agent trace
+            base=olas_data["trades"],  # Stack on top of olas trace
             showlegend=True,
         )
 
-        final_traces.extend([agent_trace, non_agent_trace])
+        final_traces.extend([olas_trace, non_Olas_trace])
 
     # Create new figure with the combined traces
     fig = go.Figure(data=final_traces)
@@ -292,7 +292,7 @@ def integrated_plot_winning_trades_per_market_by_week_v2(
     )
     # Create binary staking category
     all_filtered_trades["staking_type"] = all_filtered_trades["staking"].apply(
-        lambda x: "non_agent" if x == "non_agent" else "agent"
+        lambda x: "non_Olas" if x == "non_Olas" else "Olas"
     )
     if trader_filter == "all":
         final_df = get_overall_winning_by_market_trades(all_filtered_trades)
@@ -309,10 +309,10 @@ def integrated_plot_winning_trades_per_market_by_week_v2(
     # Convert back to string format
     all_dates = [date.strftime("%b-%d") for date in all_dates_dt]
     color_discrete_sequence = ["darkviolet", "goldenrod", "green"]
-    if trader_filter == "agent":
-        final_df = final_df[final_df["staking_type"] == "agent"]
-    elif trader_filter == "non_agent":
-        final_df = final_df[final_df["staking_type"] == "non_agent"]
+    if trader_filter == "Olas":
+        final_df = final_df[final_df["staking_type"] == "Olas"]
+    elif trader_filter == "non_Olas":
+        final_df = final_df[final_df["staking_type"] == "non_Olas"]
         color_discrete_sequence = ["purple", "darkgoldenrod", "darkgreen"]
 
     fig = px.bar(
@@ -329,58 +329,10 @@ def integrated_plot_winning_trades_per_market_by_week_v2(
         yaxis_title="Weekly % of winning trades",
         legend=dict(yanchor="top", y=0.5),
     )
-    # fig.update_layout(width=WIDTH, height=HEIGHT)
+
     fig.update_xaxes(tickformat="%b %d\n%Y")
     # Update layout to force x-axis category order (hotfix for a sorting issue)
     fig.update_layout(xaxis={"categoryorder": "array", "categoryarray": all_dates})
     return gr.Plot(
         value=fig,
     )
-
-
-def plot_winning_trades_by_week(trades_df: pd.DataFrame) -> gr.BarPlot:
-    """Plots the winning trades data for the given tools and calculates the winning percentage."""
-    return gr.BarPlot(
-        value=trades_df,
-        x="month_year_week",
-        y="winning_trade",
-        show_label=True,
-        interactive=True,
-        show_actions_button=True,
-        tooltip=["month_year_week", "winning_trade"],
-        height=HEIGHT,
-        width=WIDTH,
-    )
-
-
-def plot_winning_trades_per_market_by_week(
-    trades_df: pd.DataFrame, market_type: str
-) -> gr.Plot:
-    """Plots the winning trades data for the given tools and calculates the winning percentage."""
-    # if market_type is "all then no filter is applied"
-    if market_type == "quickstart":
-        trades = trades_df.loc[trades_df["market_creator"] == "quickstart"]
-        color_sequence = ["goldenrod"]
-
-    elif market_type == "pearl":
-        trades = trades_df.loc[trades_df["market_creator"] == "pearl"]
-        color_sequence = ["purple"]
-    else:
-        trades = trades_df
-        color_sequence = ["darkgreen"]
-
-    fig = px.bar(
-        trades,
-        x="month_year_week",
-        y="winning_trade",
-        color_discrete_sequence=color_sequence,
-        title=market_type + " winning trades",
-    )
-    fig.update_layout(
-        xaxis_title="Week",
-        yaxis_title="Weekly % of winning trades",
-    )
-    fig.update_xaxes(tickformat="%b %d\n%Y")
-    return gr.Plot(
-        value=fig,
-    )