updating dashboard data. Migrating live data

data/all_trades_profitability.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9a86faea39e8af7ea7530d6817615bc28f4bf6e2dc138af8e0f752c66142ed44
-size 1542521
+oid sha256:f1bb84f24b4dd4bef131f6f302ee4511e38ca6a02bf5991adf6dcf1333adee17
+size 1446213

data/delivers.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ea347b5596e6eab71a35ac7b59c9f158bd533c6ada77cdaee575144873ed2d5a
-size 453099936
+oid sha256:0910344b2ea12c551d29517706dbc173095ac3c209e87db052a85f61774eb285
+size 447542653

data/fpmmTrades.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2ce12cece3dcd972ed89faf643b3552e708181d18c608a3c15648638f71c27ab
-size 5198376
+oid sha256:f70bf0db5698f72afc40b17ddfcbcc129889a64e5b3f322f74c981d3a98b857d
+size 4833072

data/fpmms.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3a08a01754c17bb1212c2bdcd22fc0755d6a79bb6e1d7cd7f96014ca1a5406bc
-size 362294
+oid sha256:bf77a20572a7eda2aa3487ec6828f3e56013e9a654a4007eae65a5f40a779175
+size 368370

data/invalid_trades.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3ef965a4c367156e5d6a28ead18156108eada8c6c37304b145c0b9467369a807
-size 275567
+oid sha256:2cb0cfde1ef4547862cfe2efcdc494dba4069ec0124a5acc749c48007e4dd897
+size 244868

data/requests.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:37f54398ce420f3a67670757de1ddabd9d1c04dd0467390353984bfedb96443c
-size 13075775
+oid sha256:ac95eb71f26583a0c4a44703c757af665ae5ca243b3682f0828c96c2b9a7ace5
+size 13437238

data/summary_profitability.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6ae8a6e4f9ef7dca3f69a604c9963b1b2d0044e48fe3797a3ec88645fb55ac95
-size 38052
+oid sha256:76bc2e023eac09adc81cc337c46f023273a8aa310aed018f15506b3ba00a5d7e
+size 37419

data/t_map.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0ae0547faebf5b58075a22041049bd1674ee9b5fed961fcde7d404ec566f162c
-size 10868401
+oid sha256:795ffdc54398c79a7ed3dbf7ede11730a1fcc665098aa9f1ad6fefebb6b64ac6
+size 11186989

data/tools.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f3e7d02a408b280225d7e04c3c1f4cf1e7d96c02d7aca5314e88ff7502db347f
-size 435205780
+oid sha256:2b8f1a48a72e24c7486c7fa20895fd4b8566a4c1b3e5f00f8eab0b9e71ce8e0d
+size 448725346

data/tools_accuracy.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fb4bec695d3d6ca77611feff93cb5a2d217cbde6a70fb06561ce346a0c6f823d
-size 994
+oid sha256:d8afd4ddbff90b4232e8883484a7ffb188df18c01d667ea5dcdc4690bfce7914
+size 1007

live_data/markets_live_data.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:797e30e861c7468a5cf840a23b560d66bb9a1103d5bdae2cd86a485a43592e6e
-size 21197

live_data/markets_live_data_old.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:362255753caee81b4453f2d9c0fcf5870c770c3cf32d09c15ae6a571692a6874
-size 19052

scripts/live_markets_data.py

@@ -1,276 +0,0 @@
-#   -*- coding: utf-8 -*-
-#   ------------------------------------------------------------------------------
-#
-#     Copyright 2024 Valory AG
-#
-#     Licensed under the Apache License, Version 2.0 (the "License");
-#     you may not use this file except in compliance with the License.
-#     You may obtain a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#     Unless required by applicable law or agreed to in writing, software
-#     distributed under the License is distributed on an "AS IS" BASIS,
-#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#     See the License for the specific language governing permissions and
-#     limitations under the License.
-#
-#   ------------------------------------------------------------------------------
-
-import functools
-import warnings
-from typing import Optional, Generator, Callable
-import os
-import pandas as pd
-from datetime import datetime, timedelta, UTC
-import requests
-from tqdm import tqdm
-from typing import List, Dict
-from live_traders_data import add_trading_info
-from utils import SUBGRAPH_API_KEY, measure_execution_time
-from live_utils import OMEN_SUBGRAPH_URL, CREATOR, BATCH_SIZE, DATA_DIR
-from queries import (
-    FPMMS_WITH_TOKENS_QUERY,
-    ID_FIELD,
-    DATA_FIELD,
-    ANSWER_FIELD,
-    ANSWER_TIMESTAMP_FIELD,
-    QUERY_FIELD,
-    TITLE_FIELD,
-    OUTCOMES_FIELD,
-    OPENING_TIMESTAMP_FIELD,
-    CREATION_TIMESTAMP_FIELD,
-    LIQUIDITY_FIELD,
-    LIQUIDIY_MEASURE_FIELD,
-    TOKEN_AMOUNTS_FIELD,
-    ERROR_FIELD,
-    QUESTION_FIELD,
-    FPMMS_FIELD,
-)
-
-
-ResponseItemType = List[Dict[str, str]]
-SubgraphResponseType = Dict[str, ResponseItemType]
-
-
-class RetriesExceeded(Exception):
-    """Exception to raise when retries are exceeded during data-fetching."""
-
-    def __init__(
-        self, msg="Maximum retries were exceeded while trying to fetch the data!"
-    ):
-        super().__init__(msg)
-
-
-def hacky_retry(func: Callable, n_retries: int = 3) -> Callable:
-    """Create a hacky retry strategy.
-        Unfortunately, we cannot use `requests.packages.urllib3.util.retry.Retry`,
-        because the subgraph does not return the appropriate status codes in case of failure.
-        Instead, it always returns code 200. Thus, we raise exceptions manually inside `make_request`,
-        catch those exceptions in the hacky retry decorator and try again.
-        Finally, if the allowed number of retries is exceeded, we raise a custom `RetriesExceeded` exception.
-
-    :param func: the input request function.
-    :param n_retries: the maximum allowed number of retries.
-    :return: The request method with the hacky retry strategy applied.
-    """
-
-    @functools.wraps(func)
-    def wrapper_hacky_retry(*args, **kwargs) -> SubgraphResponseType:
-        """The wrapper for the hacky retry.
-
-        :return: a response dictionary.
-        """
-        retried = 0
-
-        while retried <= n_retries:
-            try:
-                if retried > 0:
-                    warnings.warn(f"Retrying {retried}/{n_retries}...")
-
-                return func(*args, **kwargs)
-            except (ValueError, ConnectionError) as e:
-                warnings.warn(e.args[0])
-            finally:
-                retried += 1
-
-        raise RetriesExceeded()
-
-    return wrapper_hacky_retry
-
-
-@hacky_retry
-def query_subgraph(url: str, query: str, key: str) -> SubgraphResponseType:
-    """Query a subgraph.
-
-    Args:
-        url: the subgraph's URL.
-        query: the query to be used.
-        key: the key to use in order to access the required data.
-
-    Returns:
-        a response dictionary.
-    """
-    content = {QUERY_FIELD: query}
-    headers = {
-        "Accept": "application/json",
-        "Content-Type": "application/json",
-    }
-    res = requests.post(url, json=content, headers=headers)
-
-    if res.status_code != 200:
-        raise ConnectionError(
-            "Something went wrong while trying to communicate with the subgraph "
-            f"(Error: {res.status_code})!\n{res.text}"
-        )
-
-    body = res.json()
-    if ERROR_FIELD in body.keys():
-        raise ValueError(f"The given query is not correct: {body[ERROR_FIELD]}")
-
-    data = body.get(DATA_FIELD, {}).get(key, None)
-    if data is None:
-        raise ValueError(f"Unknown error encountered!\nRaw response: \n{body}")
-
-    return data
-
-
-def fpmms_fetcher(current_timestamp: int) -> Generator[ResponseItemType, int, None]:
-    """An indefinite fetcher for the FPMMs."""
-    omen_subgraph = OMEN_SUBGRAPH_URL.substitute(subgraph_api_key=SUBGRAPH_API_KEY)
-    print(f"omen_subgraph = {omen_subgraph}")
-    while True:
-        fpmm_id = yield
-        fpmms_query = FPMMS_WITH_TOKENS_QUERY.substitute(
-            creator=CREATOR,
-            fpmm_id=fpmm_id,
-            current_timestamp=current_timestamp,
-            fpmms_field=FPMMS_FIELD,
-            first=BATCH_SIZE,
-            id_field=ID_FIELD,
-            answer_timestamp_field=ANSWER_TIMESTAMP_FIELD,
-            question_field=QUESTION_FIELD,
-            outcomes_field=OUTCOMES_FIELD,
-            title_field=TITLE_FIELD,
-            opening_timestamp_field=OPENING_TIMESTAMP_FIELD,
-            creation_timestamp_field=CREATION_TIMESTAMP_FIELD,
-            liquidity_field=LIQUIDITY_FIELD,
-            liquidity_measure_field=LIQUIDIY_MEASURE_FIELD,
-            token_amounts_field=TOKEN_AMOUNTS_FIELD,
-        )
-        print(f"Executing query {fpmms_query}")
-        yield query_subgraph(omen_subgraph, fpmms_query, FPMMS_FIELD)
-
-
-def fetch_fpmms(current_timestamp: int) -> pd.DataFrame:
-    """Fetch all the fpmms of the creator."""
-    print("Fetching all markets")
-    latest_id = ""
-    fpmms = []
-    fetcher = fpmms_fetcher(current_timestamp)
-    for _ in tqdm(fetcher, unit="fpmms", unit_scale=BATCH_SIZE):
-        batch = fetcher.send(latest_id)
-        if len(batch) == 0:
-            print("no data")
-            break
-
-        # TODO Add the incremental batching system from market creator
-        # prev_fpmms is the previous local file with the markets
-        # for fpmm in batch:
-        #     if fpmm["id"] not in fpmms or "trades" not in prev_fpmms[fpmm["id"]]:
-        #         prev_fpmms[fpmm["id"]] = fpmm
-        print(f"length of the data received = {len(batch)}")
-        latest_id = batch[-1].get(ID_FIELD, "")
-        if latest_id == "":
-            raise ValueError(f"Unexpected data format retrieved: {batch}")
-
-        fpmms.extend(batch)
-
-    print("Finished collecting data")
-    return pd.DataFrame(fpmms)
-
-
-def get_answer(fpmm: pd.Series) -> str:
-    """Get an answer from its index, using Series of an FPMM."""
-    return fpmm[QUESTION_FIELD][OUTCOMES_FIELD][fpmm[ANSWER_FIELD]]
-
-
-def get_first_token_perc(row):
-    if row["total_tokens"] == 0.0:
-        return 0
-    return round((row["token_first_amount"] / row["total_tokens"]) * 100, 2)
-
-
-def get_second_token_perc(row):
-    if row["total_tokens"] == 0.0:
-        return 0
-    return round((row["token_second_amount"] / row["total_tokens"]) * 100, 2)
-
-
-def transform_fpmms(fpmms: pd.DataFrame, filename: str, current_timestamp: int) -> None:
-    """Transform an FPMMS dataframe."""
-
-    # prepare the new ones
-    # Add current timestamp
-    fpmms["tokens_timestamp"] = current_timestamp
-    fpmms["open"] = True
-
-    # computation of token distributions
-    fpmms["token_first_amount"] = fpmms.outcomeTokenAmounts.apply(lambda x: int(x[0]))
-    fpmms["token_second_amount"] = fpmms.outcomeTokenAmounts.apply(lambda x: int(x[1]))
-    fpmms["total_tokens"] = fpmms.apply(
-        lambda x: x.token_first_amount + x.token_second_amount, axis=1
-    )
-    fpmms["first_token_perc"] = fpmms.apply(lambda x: get_first_token_perc(x), axis=1)
-    fpmms["second_token_perc"] = fpmms.apply(lambda x: get_second_token_perc(x), axis=1)
-    fpmms.drop(
-        columns=["token_first_amount", "token_second_amount", "total_tokens"],
-        inplace=True,
-    )
-    # previous file to update?
-    old_fpmms = None
-    if os.path.exists(DATA_DIR / filename):
-        old_fpmms = pd.read_parquet(DATA_DIR / filename)
-
-    if old_fpmms is not None:
-        # update which markets are not open anymore
-        open_markets = list(fpmms.id.unique())
-        print("Updating market status of old markets")
-        open_mask = old_fpmms["id"].isin(open_markets)
-        old_fpmms.loc[~open_mask, "status"] = False
-
-        # now concatenate
-        print("Appending new data to previous data")
-        fpmms = pd.concat([old_fpmms, fpmms], ignore_index=True)
-        # fpmms.drop_duplicates(inplace=True)
-
-    return
-
-
-@measure_execution_time
-def compute_distributions(filename: Optional[str]) -> pd.DataFrame:
-    """Fetch, process, store and return the markets as a Dataframe."""
-
-    print("fetching new markets information")
-    current_timestamp = int(datetime.now(UTC).timestamp())
-    fpmms = fetch_fpmms(current_timestamp)
-    print(fpmms.head())
-
-    print("transforming and updating previous data")
-
-    transform_fpmms(fpmms, filename, current_timestamp)
-    print(fpmms.head())
-
-    # WIP
-    # print("Adding trading information")
-    add_trading_info(fpmms)
-    print("saving the data")
-    print(fpmms.info())
-    if filename:
-        fpmms.to_parquet(DATA_DIR / filename, index=False)
-
-    return fpmms
-
-
-if __name__ == "__main__":
-    compute_distributions("markets_live_data.parquet")

scripts/live_traders_data.py

@@ -1,153 +0,0 @@
-import requests
-import os
-from datetime import datetime, timedelta, UTC
-import pandas as pd
-from collections import defaultdict
-from typing import Any, Optional
-from tqdm import tqdm
-from live_utils import OMEN_SUBGRAPH_URL, CREATOR, BATCH_SIZE, DATA_DIR
-from utils import SUBGRAPH_API_KEY, _to_content
-from queries import omen_trader_votes_query
-
-
-headers = {
-    "Accept": "application/json, multipart/mixed",
-    "Content-Type": "application/json",
-}
-
-
-def _query_omen_xdai_subgraph(
-    fpmm_id: str,
-) -> dict[str, Any]:
-    """Query the subgraph."""
-    omen_subgraph = OMEN_SUBGRAPH_URL.substitute(subgraph_api_key=SUBGRAPH_API_KEY)
-    print(f"omen_subgraph = {omen_subgraph}")
-    grouped_results = defaultdict(list)
-    id_gt = ""
-
-    while True:
-        query = omen_trader_votes_query.substitute(
-            fpmm_creator=CREATOR.lower(),
-            first=BATCH_SIZE,
-            id_gt=id_gt,
-            fpmm_id=fpmm_id,
-        )
-        print(f"query for the omen to collect trades {query}")
-        content_json = _to_content(query)
-
-        res = requests.post(omen_subgraph, headers=headers, json=content_json)
-        result_json = res.json()
-        # print(f"result = {result_json}")
-        user_trades = result_json.get("data", {}).get("fpmmTrades", [])
-
-        if not user_trades:
-            break
-
-        for trade in user_trades:
-            fpmm_id = trade.get("fpmm", {}).get("id")
-            grouped_results[fpmm_id].append(trade)
-
-        id_gt = user_trades[len(user_trades) - 1]["id"]
-
-    all_results = {
-        "data": {
-            "fpmmTrades": [
-                trade
-                for trades_list in grouped_results.values()
-                for trade in trades_list
-            ]
-        }
-    }
-
-    return all_results
-
-
-def transform_trades(trades_json: dict) -> pd.DataFrame:
-    # convert to dataframe
-    print("transforming trades")
-    df = pd.DataFrame(trades_json["data"]["fpmmTrades"])
-    if len(df) == 0:
-        print("No trades for this market")
-        return df
-
-    # print(df.info())
-
-    # convert creator to address
-    df["trade_creator"] = df["creator"].apply(lambda x: x["id"])
-
-    # normalize fpmm column
-    fpmm = pd.json_normalize(df["fpmm"])
-    fpmm.columns = [f"fpmm.{col}" for col in fpmm.columns]
-    df = pd.concat([df, fpmm], axis=1)
-
-    # drop fpmm column
-    df.drop(["fpmm"], axis=1, inplace=True)
-
-    # convert into int
-    df.outcomeIndex = pd.to_numeric(df.outcomeIndex, errors="coerce")
-    return df
-
-
-def compute_from_timestamp_value(
-    fpmm_id: str, opening_timestamp: int, fpmms: pd.DataFrame
-) -> Optional[int]:
-    """Function to find the latest timestamp registered for a specific market"""
-    try:
-        market_data = fpmms.loc[fpmms["id"] == fpmm_id]
-        # how many previous samples do we have?
-        if len(market_data) == 1:
-            # take the opening Timestamp of the Market
-            return opening_timestamp
-        timestamps = (market_data.tokens_timestamp.values).sort()
-        # the last value is the current timestamp so we need to take the previous one
-        return timestamps[-2]
-    except Exception as e:
-        print(
-            f"Error when trying to get the from timestamp value of the market id {fpmm_id}"
-        )
-        return None
-
-
-def compute_votes_distribution(market_trades: pd.DataFrame):
-    """Function to compute the distribution of votes for the trades of a market"""
-    total_trades = len(market_trades)
-    print(f"The total number of trades is {total_trades}")
-    # outcomeIndex is always 1 or 0?
-    sum_outcome_index_1 = sum(market_trades.outcomeIndex)
-    print(f"The total number of votes for index 1 is {sum_outcome_index_1}")
-    percentage_index_1 = round((sum_outcome_index_1 / total_trades) * 100, 2)
-    return (100 - percentage_index_1), percentage_index_1
-
-
-def add_trading_info(fpmms: pd.DataFrame) -> None:
-    # Iterate over the markets
-    print("Adding votes distribution per market")
-    fpmms["votes_first_outcome_perc"] = 0.0
-    fpmms["votes_second_outcome_perc"] = 0.0
-    for i, fpmm in tqdm(fpmms.iterrows(), total=len(fpmms), desc="Analysing trades"):
-        # read trades from latest read timestamp
-        market_id = fpmm["id"]
-        print(f"Adding information for the market {market_id}")
-        market_trades_json = _query_omen_xdai_subgraph(
-            fpmm_id=market_id,
-        )
-        market_trades = transform_trades(market_trades_json)
-        if len(market_trades) == 0:
-            continue
-        # to compute the votes distribution
-        print("Computing the votes distribution")
-        first_outcome, second_outcome = compute_votes_distribution(market_trades)
-        print(
-            f"first outcome votes ={first_outcome}, second outcome votes = {second_outcome}"
-        )
-        fpmms.loc[fpmms["id"] == market_id, "votes_first_outcome_perc"] = first_outcome
-        fpmms.loc[fpmms["id"] == market_id, "votes_second_outcome_perc"] = (
-            second_outcome
-        )
-    print("Dataset after adding trading info")
-    print(fpmms.head())
-    return
-
-
-if __name__ == "__main__":
-    print("collecting votes distribution")

scripts/live_utils.py

@@ -1,14 +0,0 @@
-from string import Template
-from pathlib import Path
-
-
-CREATOR = "0x89c5cc945dd550BcFfb72Fe42BfF002429F46Fec"
-BATCH_SIZE = 1000
-# OMEN_SUBGRAPH = "https://api.thegraph.com/subgraphs/name/protofire/omen-xdai"
-OMEN_SUBGRAPH_URL = Template(
-    """https://gateway-arbitrum.network.thegraph.com/api/${subgraph_api_key}/subgraphs/id/9fUVQpFwzpdWS9bq5WkAnmKbNNcoBwatMR4yZq81pbbz"""
-)
-SCRIPTS_DIR = Path(__file__).parent
-ROOT_DIR = SCRIPTS_DIR.parent
-DATA_DIR = ROOT_DIR / "live_data"
-MAX_UINT_HEX = "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"

scripts/queries.py

@@ -59,39 +59,6 @@ FPMMS_QUERY = Template(
     """
 )
 
-FPMMS_WITH_TOKENS_QUERY = Template(
-    """
-    {
-      ${fpmms_field}(
-        where: {
-          creator: "${creator}",
-          id_gt: "${fpmm_id}",
-          isPendingArbitration: false
-          currentAnswer: null
-          openingTimestamp_gt:${current_timestamp}
-        },
-        orderBy: ${id_field}
-        orderDirection: asc
-        first: ${first}
-      ){
-        ${id_field}
-        ${question_field} {
-          ${outcomes_field}
-          ${answer_timestamp_field}
-          answers{
-          answer
-          }
-        }
-        ${title_field}
-        ${opening_timestamp_field}
-        ${creation_timestamp_field}
-        ${liquidity_field}
-        ${liquidity_measure_field}
-        ${token_amounts_field}
-      }
-    }
-    """
-)
 omen_xdai_trades_query = Template(
     """
     {
@@ -145,52 +112,6 @@ omen_xdai_trades_query = Template(
     """
 )
 
-omen_trader_votes_query = Template(
-    """
-    {
-        fpmmTrades(
-            where: {
-                type: Buy,
-                fpmm_: {
-                    creator: "${fpmm_creator}",
-                    id: "${fpmm_id}",
-                },
-                id_gt: "${id_gt}"
-            }
-            first: ${first}
-            orderBy: id
-            orderDirection: asc
-        ) {
-            id
-            title
-            collateralToken
-            outcomeTokenMarginalPrice
-            oldOutcomeTokenMarginalPrice
-            type
-            creator {
-                id
-            }
-            creationTimestamp
-            collateralAmount
-            collateralAmountUSD
-            feeAmount
-            outcomeIndex
-            outcomeTokensTraded
-            transactionHash
-            fpmm {
-                id
-                outcomes
-                title
-                condition {
-                    id
-                }
-            }
-        }
-    }
-    """
-)
-
-
 conditional_tokens_gc_user_query = Template(
     """
     {