updating daily info

app.py

@@ -7,11 +7,8 @@ from tabs.trades import (
     prepare_trades,
     get_overall_trades,
     get_overall_by_market_trades,
-    get_overall_winning_trades,
     get_overall_winning_by_market_trades,
-    integrated_plot_trades_per_market_by_week,
     integrated_plot_trades_per_market_by_week_v2,
-    integrated_plot_winning_trades_per_market_by_week,
     integrated_plot_winning_trades_per_market_by_week_v2,
 )
 from tabs.staking import plot_staking_trades_per_market_by_week
@@ -173,7 +170,7 @@ def prepare_data():
 
 
 tools_df, trades_df, tools_accuracy_info, invalid_trades = prepare_data()
-
+trades_df = trades_df.sort_values(by="creation_timestamp", ascending=True)
 
 demo = gr.Blocks()
 
@@ -184,7 +181,6 @@ error_overall_by_markets = get_error_data_overall_by_market(error_df=error_by_ma
 winning_df = get_tool_winning_rate_by_market(tools_df, inc_tools=INC_TOOLS)
 # preparing data for the trades graph
 trades_count_df = get_overall_trades(trades_df=trades_df)
-trades_winning_rate_df = get_overall_winning_trades(trades_df=trades_df)
 trades_by_market = get_overall_by_market_trades(trades_df=trades_df)
 winning_trades_by_market = get_overall_winning_by_market_trades(trades_df=trades_df)
 with demo:

data/all_trades_profitability.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:223f85e66279e8e12547e53f16efb0af7c9c902578b1cc529c878f7ee7379ce6
-size 3551233
+oid sha256:52f72088df284d5addddb5d6dd3e2226c8fc98c58b9048c0cf145d52016da783
+size 3136886

data/daily_info.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f41fdb6fc36cf8cb28980bca049de6b4fa986a9800176e269a5259a7e744c514
-size 251792
+oid sha256:aa46e26ddf79ae4565e8572d489d377884e071f20eb19ea3c8d99684c2c00548
+size 611323

data/fpmmTrades.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4a55fcd7ac6fe8e8008fd8395edd39cc3ac0a05ab202f4296c22943f42f33470
-size 18283162
+oid sha256:82b8e872c454048229fd0fc85ddd3b848d9d52b6e336113f239b8ed90d745141
+size 19295640

data/fpmms.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:61ce94d0b622f72f695a244890bf91ae6286e793bd65cb5393a3d8bc810ff1a5
-size 525473
+oid sha256:bcee66dc9b6b8f6be7e8cd33f57344f126af01c503c9e08f05d571b0157109f5
+size 529819

data/new_fpmmTrades.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2179890c22497ae3572341fed5eb8b73bddc90f27c7b5c60037edd8170e74e51
-size 4542500
+oid sha256:f8bf44ce187a57486f821f934fbb8f6676b14b4c4a1a1d25806c5cf6255614aa
+size 2265950

data/new_tools.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6afba2f5c6e14a81abbf1f936ddb0f83ada76c4515a6daf86fbc01ceba1a6e32
-size 79645959
+oid sha256:6b4d38de15b4da119a9706cc5addd45f1979bac11eac70442f45155805a9d5bc
+size 25083815

data/outliers.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:494d9e9d0a20fcac4699dd4bf7bc658a50a124e8e33060997dd5eed0ce4863bc
-size 18711
+oid sha256:81f07e9f5a1ad5c39b73068888e94260f86782f4f9511cbf548cd366ff827218
+size 19361

data/summary_profitability.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8ef6ed929a47113c2afac034356a094b8af089bb03f9e5a673c6cc2f27978120
-size 94924
+oid sha256:3047c4a75c6c90fb8132d625f7501d979ebb1fb5711ede766fa66009c6dad5e3
+size 92478

data/t_map.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ed3bcdd7a1c5ed48707a5fdb0dd16775d2ba347c7416fe6d4b5dc3a71dfea212
-size 25526067
+oid sha256:45a4dd3205c972655bece22f01445b44cf9f445a41a8492f60083a22bd734a95
+size 25661723

data/tools.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5e4acfa4aa1c8c80150269caf40fe6ff16e7aff3c90e2c4ca809d29357a6c36b
-size 448917044
+oid sha256:87d50073d2f75f1a5f09353ac46747180046ade9be74cf8bb52167e30da2085d
+size 446483909

data/tools_accuracy.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:174f3deae7eb082b6623e409771f17f538e94f74bb4fcfa62ea211c60ff3404a
-size 1339
+oid sha256:c6c7f4a8a992798d4920949a1e1d839474512a6b291b3fe19138d8f921574253
+size 1325

historical_data/all_trades_profitability_20241128_145606.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:223f85e66279e8e12547e53f16efb0af7c9c902578b1cc529c878f7ee7379ce6
+size 3551233

historical_data/tools_20241128_145606.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:993153ec73833d33a2c28499837ee0547a9e92fdd598766faccdfff0a6eced18
+size 488681078

scripts/cleaning_old_info.py

@@ -7,6 +7,7 @@ from staking import label_trades_by_staking
 
 
 def clean_old_data_from_parquet_files(cutoff_date: str):
+    print("Cleaning oldest data")
     # Convert the string to datetime64[ns, UTC]
     min_date_utc = pd.to_datetime(cutoff_date, format="%Y-%m-%d", utc=True)
 

scripts/pull_data.py

@@ -28,6 +28,7 @@ logging.basicConfig(level=logging.INFO)
 SCRIPTS_DIR = Path(__file__).parent
 ROOT_DIR = SCRIPTS_DIR.parent
 DATA_DIR = ROOT_DIR / "data"
+HIST_DIR = ROOT_DIR / "historical_data"
 
 
 def block_number_to_timestamp(block_number: int, web3: Web3) -> str:
@@ -119,6 +120,33 @@ def updating_timestamps(rpc: str, tools_filename: str):
     gc.collect()
 
 
+def save_historical_data():
+    """Function to save a copy of the main trades and tools file
+    into the historical folder"""
+    print("Saving historical data copies")
+    current_datetime = datetime.now()
+
+    timestamp = current_datetime.strftime("%Y%m%d_%H%M%S")
+
+    try:
+        tools = pd.read_parquet(DATA_DIR / "tools.parquet")
+        filename = f"tools_{timestamp}.parquet"
+        tools.to_parquet(HIST_DIR / filename, index=False)
+
+    except Exception as e:
+        print(f"Error saving tools file in the historical folder {e}")
+
+    try:
+        all_trades = pd.read_parquet(DATA_DIR / "all_trades_profitability.parquet")
+        filename = f"all_trades_profitability_{timestamp}.parquet"
+        all_trades.to_parquet(HIST_DIR / filename, index=False)
+
+    except Exception as e:
+        print(
+            f"Error saving all_trades_profitability file in the historical folder {e}"
+        )
+
+
 @measure_execution_time
 def only_new_weekly_analysis():
     """Run weekly analysis for the FPMMS project."""
@@ -166,7 +194,9 @@ def only_new_weekly_analysis():
         logging.error("Error while updating timestamps of tools")
         print(e)
 
-    clean_old_data_from_parquet_files("2024-09-22")
+    save_historical_data()
+
+    clean_old_data_from_parquet_files("2024-09-29")
 
     compute_tools_accuracy()
 

tabs/metrics.py

@@ -2,6 +2,7 @@ import pandas as pd
 import gradio as gr
 import plotly.express as px
 import gc
+from datetime import datetime
 
 trade_metric_choices = [
     "mech calls",
@@ -155,6 +156,15 @@ def plot_trade_metrics(
         )
     else:
         trades_filtered = get_boxplot_metrics(column_name, trades_df)
+    # Convert string dates to datetime and sort them
+    all_dates_dt = sorted(
+        [
+            datetime.strptime(date, "%b-%d")
+            for date in trades_filtered["month_year_week"].unique()
+        ]
+    )
+    # Convert back to string format
+    all_dates = [date.strftime("%b-%d") for date in all_dates_dt]
     fig = px.box(
         trades_filtered,
         x="month_year_week",
@@ -170,33 +180,13 @@ def plot_trade_metrics(
         legend=dict(yanchor="top", y=0.5),
     )
     fig.update_xaxes(tickformat="%b %d\n%Y")
+    # Update layout to force x-axis category order (hotfix for a sorting issue)
+    fig.update_layout(xaxis={"categoryorder": "array", "categoryarray": all_dates})
     return gr.Plot(
         value=fig,
     )
 
 
-def plot_average_roi_per_market_by_week(trades_df: pd.DataFrame) -> gr.LinePlot:
-
-    mean_roi_per_market_by_week = (
-        trades_df.groupby(["market_creator", "month_year_week"])["roi"]
-        .mean()
-        .reset_index()
-    )
-    mean_roi_per_market_by_week.rename(columns={"roi": "mean_roi"}, inplace=True)
-    return gr.LinePlot(
-        value=mean_roi_per_market_by_week,
-        x="month_year_week",
-        y="ROI",
-        color="market_creator",
-        show_label=True,
-        interactive=True,
-        show_actions_button=True,
-        tooltip=["month_year_week", "market_creator", "mean_roi"],
-        height=HEIGHT,
-        width=WIDTH,
-    )
-
-
 def get_trade_metrics_text() -> gr.Markdown:
     metric_text = """ 
         ## Description of the graph

tabs/staking.py

@@ -1,6 +1,7 @@
 import gradio as gr
 import pandas as pd
 import plotly.express as px
+from datetime import datetime
 
 
 def get_overall_by_staking_traders(trades_df: pd.DataFrame) -> pd.DataFrame:
@@ -24,7 +25,6 @@ def plot_staking_trades_per_market_by_week(
     trades_all["market_creator"] = "all"
 
     # choose colour
-
     market_colour = "green"
     if market_creator == "pearl":
         market_colour = "darkviolet"
@@ -36,11 +36,9 @@ def plot_staking_trades_per_market_by_week(
     all_filtered_trades = all_filtered_trades.sort_values(
         by="creation_timestamp", ascending=True
     )
-    print(f"filtering by market creator = {market_creator}")
     all_filtered_trades = all_filtered_trades.loc[
         all_filtered_trades["market_creator"] == market_creator
     ]
-    print(all_filtered_trades.market_creator.value_counts())
 
     if market_creator != "all":
         all_filtered_trades["staking"] = all_filtered_trades["staking"].replace(
@@ -68,6 +66,16 @@ def plot_staking_trades_per_market_by_week(
             ]
         }
     trades = get_overall_by_staking_traders(all_filtered_trades)
+    # Convert string dates to datetime and sort them
+    all_dates_dt = sorted(
+        [
+            datetime.strptime(date, "%b-%d")
+            for date in trades["month_year_week"].unique()
+        ]
+    )
+    # Convert back to string format
+    all_dates = [date.strftime("%b-%d") for date in all_dates_dt]
+
     fig = px.bar(
         trades,
         x="month_year_week",
@@ -86,4 +94,6 @@ def plot_staking_trades_per_market_by_week(
         height=600,  # Adjusted for better fit on laptop screens
     )
     fig.update_xaxes(tickformat="%b %d\n%Y")
+    # Update layout to force x-axis category order (hotfix for a sorting issue)
+    fig.update_layout(xaxis={"categoryorder": "array", "categoryarray": all_dates})
     return gr.Plot(value=fig)

tabs/trades.py

@@ -3,7 +3,7 @@ import pandas as pd
 import plotly.express as px
 import plotly.graph_objects as go
 from plotly.subplots import make_subplots
-
+from datetime import datetime
 
 HEIGHT = 400
 WIDTH = 1100
@@ -163,7 +163,6 @@ def integrated_plot_trades_per_market_by_week_v2(trades_df: pd.DataFrame) -> gr.
     all_filtered_trades = all_filtered_trades.sort_values(
         by="creation_timestamp", ascending=True
     )
-
     # Create binary staking category
     all_filtered_trades["staking_type"] = all_filtered_trades["staking"].apply(
         lambda x: "non_agent" if x == "non_agent" else "agent"
@@ -177,7 +176,15 @@ def integrated_plot_trades_per_market_by_week_v2(trades_df: pd.DataFrame) -> gr.
         .size()
         .reset_index(name="trades")
     )
-
+    # Convert string dates to datetime and sort them
+    all_dates_dt = sorted(
+        [
+            datetime.strptime(date, "%b-%d")
+            for date in trades["month_year_week"].unique()
+        ]
+    )
+    # Convert back to string format
+    all_dates = [date.strftime("%b-%d") for date in all_dates_dt]
     # Combine the traces
     final_traces = []
     market_colors = {"pearl": "darkviolet", "quickstart": "goldenrod", "all": "green"}
@@ -185,7 +192,6 @@ def integrated_plot_trades_per_market_by_week_v2(trades_df: pd.DataFrame) -> gr.
     for market in ["pearl", "quickstart", "all"]:
         market_data = trades[trades["market_creator"] == market]
         agent_data = market_data[market_data["staking_type"] == "agent"]
-
         trace = go.Bar(
             x=agent_data["month_year_week"],
             y=agent_data["trades"],
@@ -205,7 +211,6 @@ def integrated_plot_trades_per_market_by_week_v2(trades_df: pd.DataFrame) -> gr.
     for market in ["pearl", "quickstart", "all"]:
         market_data = trades[trades["market_creator"] == market]
         non_agent_data = market_data[market_data["staking_type"] == "non_agent"]
-
         trace = go.Bar(
             x=non_agent_data["month_year_week"],
             y=non_agent_data["trades"],
@@ -231,6 +236,8 @@ def integrated_plot_trades_per_market_by_week_v2(trades_df: pd.DataFrame) -> gr.
 
     # Update x-axis format
     fig.update_xaxes(tickformat="%b %d\n%Y")
+    # Update layout to force x-axis category order (hotfix for a sorting issue)
+    fig.update_layout(xaxis={"categoryorder": "array", "categoryarray": all_dates})
 
     return gr.Plot(value=fig)
 
@@ -270,7 +277,7 @@ def integrated_plot_winning_trades_per_market_by_week(
 
 
 def integrated_plot_winning_trades_per_market_by_week_v2(
-    trades_df: pd.DataFrame, trader_filter: str = None
+    trades_df: pd.DataFrame, trader_filter: str = "all"
 ) -> gr.Plot:
     # adding the total
     trades_all = trades_df.copy(deep=True)
@@ -285,11 +292,20 @@ def integrated_plot_winning_trades_per_market_by_week_v2(
     all_filtered_trades["staking_type"] = all_filtered_trades["staking"].apply(
         lambda x: "non_agent" if x == "non_agent" else "agent"
     )
-    if trader_filter is None:
+    if trader_filter == "all":
         final_df = get_overall_winning_by_market_trades(all_filtered_trades)
     else:
         final_df = get_overall_winning_by_market_and_trader_type(all_filtered_trades)
 
+    # Convert string dates to datetime and sort them
+    all_dates_dt = sorted(
+        [
+            datetime.strptime(date, "%b-%d")
+            for date in final_df["month_year_week"].unique()
+        ]
+    )
+    # Convert back to string format
+    all_dates = [date.strftime("%b-%d") for date in all_dates_dt]
     color_discrete_sequence = ["darkviolet", "goldenrod", "green"]
     if trader_filter == "agent":
         final_df = final_df[final_df["staking_type"] == "agent"]
@@ -313,6 +329,8 @@ def integrated_plot_winning_trades_per_market_by_week_v2(
     )
     # fig.update_layout(width=WIDTH, height=HEIGHT)
     fig.update_xaxes(tickformat="%b %d\n%Y")
+    # Update layout to force x-axis category order (hotfix for a sorting issue)
+    fig.update_layout(xaxis={"categoryorder": "array", "categoryarray": all_dates})
     return gr.Plot(
         value=fig,
     )