File size: 5,445 Bytes
04a2c17 9382590 b3c2f09 964692a 9382590 964692a 1df3214 26538e1 b3c2f09 04a2c17 5f5eb85 b3c2f09 04a2c17 9382590 04a2c17 ac347d2 9382590 04a2c17 5f5eb85 04a2c17 5f5eb85 10a7862 9382590 10a7862 da55889 681e0a4 9382590 681e0a4 9382590 26538e1 9382590 3859cbd 9382590 da55889 b3c2f09 26538e1 9382590 b3c2f09 544f140 10a7862 544f140 964692a 9382590 964692a 544f140 964692a dbb4d06 9382590 544f140 9382590 544f140 9382590 544f140 9382590 10a7862 9382590 10a7862 544f140 1df3214 04a2c17 3859cbd 10a7862 1df3214 964692a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 |
import logging
from datetime import datetime
import pandas as pd
from markets import (
etl as mkt_etl,
DEFAULT_FILENAME as MARKETS_FILENAME,
)
from tools import DEFAULT_FILENAME as TOOLS_FILENAME, generate_tools_file
from profitability import run_profitability_analysis, DEFAULT_60_DAYS_AGO_TIMESTAMP
from utils import (
get_question,
current_answer,
RPC,
measure_execution_time,
DATA_DIR,
HIST_DIR,
)
from get_mech_info import (
get_mech_events_last_60_days,
get_mech_events_since_last_run,
update_json_files,
)
from update_tools_accuracy import compute_tools_accuracy
from cleaning_old_info import clean_old_data_from_parquet_files
from web3_utils import updating_timestamps
logging.basicConfig(level=logging.INFO)
def add_current_answer(tools_filename: str):
# Get currentAnswer from FPMMS
fpmms = pd.read_parquet(DATA_DIR / MARKETS_FILENAME)
tools = pd.read_parquet(DATA_DIR / tools_filename)
# Get the question from the tools
logging.info("Getting the question and current answer for the tools")
tools["title"] = tools["prompt_request"].apply(lambda x: get_question(x))
tools["currentAnswer"] = tools["title"].apply(lambda x: current_answer(x, fpmms))
tools["currentAnswer"] = tools["currentAnswer"].str.replace("yes", "Yes")
tools["currentAnswer"] = tools["currentAnswer"].str.replace("no", "No")
# Save the tools data after the updates on the content
tools.to_parquet(DATA_DIR / tools_filename, index=False)
del fpmms
def save_historical_data():
"""Function to save a copy of the main trades and tools file
into the historical folder"""
print("Saving historical data copies")
current_datetime = datetime.now()
timestamp = current_datetime.strftime("%Y%m%d_%H%M%S")
try:
tools = pd.read_parquet(DATA_DIR / "tools.parquet")
filename = f"tools_{timestamp}.parquet"
tools.to_parquet(HIST_DIR / filename, index=False)
except Exception as e:
print(f"Error saving tools file in the historical folder {e}")
try:
all_trades = pd.read_parquet(DATA_DIR / "all_trades_profitability.parquet")
filename = f"all_trades_profitability_{timestamp}.parquet"
all_trades.to_parquet(HIST_DIR / filename, index=False)
except Exception as e:
print(
f"Error saving all_trades_profitability file in the historical folder {e}"
)
@measure_execution_time
def only_new_weekly_analysis():
"""Run weekly analysis for the FPMMS project."""
rpc = RPC
# Run markets ETL
logging.info("Running markets ETL")
mkt_etl(MARKETS_FILENAME)
logging.info("Markets ETL completed")
# New tools ETL
logging.info("Generating the mech json files")
# get only new data
latest_timestamp = get_mech_events_since_last_run()
if latest_timestamp == None:
print("Error while getting the mech events")
return
logging.info(f"Finished generating the mech json files from {latest_timestamp}")
# Run tools ETL
logging.info("Generate and parse the tools content")
# generate only new file
generate_tools_file("new_tools_info.json", "new_tools.parquet")
logging.info("Tools ETL completed")
add_current_answer("new_tools.parquet")
# # Run profitability analysis
logging.info("Running profitability analysis")
run_profitability_analysis(
rpc=rpc,
tools_filename="new_tools.parquet",
trades_filename="new_fpmmTrades.parquet",
from_timestamp=int(latest_timestamp.timestamp()),
merge=True,
)
logging.info("Profitability analysis completed")
# merge new json files with old json files
update_json_files()
try:
updating_timestamps(rpc, TOOLS_FILENAME)
except Exception as e:
logging.error("Error while updating timestamps of tools")
print(e)
save_historical_data()
clean_old_data_from_parquet_files("2024-10-06")
compute_tools_accuracy()
logging.info("Weekly analysis files generated and saved")
def old_weekly_analysis():
"""Run weekly analysis for the FPMMS project."""
rpc = RPC
# Run markets ETL
logging.info("Running markets ETL")
mkt_etl(MARKETS_FILENAME)
logging.info("Markets ETL completed")
# New tools ETL
logging.info("Generating the mech json files")
get_mech_events_last_60_days()
logging.info("Finished generating the mech json files")
# Run tools ETL
logging.info("Generate and parse the tools content")
generate_tools_file("tools_info.json", TOOLS_FILENAME)
logging.info("Tools ETL completed")
add_current_answer(TOOLS_FILENAME)
# Run profitability analysis
logging.info("Running profitability analysis")
run_profitability_analysis(
rpc=rpc,
tools_filename=TOOLS_FILENAME,
trades_filename="fpmmTrades.parquet",
from_timestamp=DEFAULT_60_DAYS_AGO_TIMESTAMP,
)
logging.info("Profitability analysis completed")
try:
updating_timestamps(rpc, TOOLS_FILENAME)
except Exception as e:
logging.error("Error while updating timestamps of tools")
print(e)
compute_tools_accuracy()
logging.info("Weekly analysis files generated and saved")
if __name__ == "__main__":
only_new_weekly_analysis()
# weekly_analysis()
# rpc = RPC
# updating_timestamps(rpc)
# compute_tools_accuracy()
|