rosacastillo commited on
Commit
10a7862
·
1 Parent(s): 580ec7b

Added RPC env variable. New updated data

Browse files
data/all_trades_profitability.parquet CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:54a498cf077943ef9e50a6c8359e43b5577f6377dc81889b129b6c8964f9987f
3
- size 1522788
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:93705929fc62c990a18bcd01e5524328a4d514ecc6fea1ba2f4ce31563316abb
3
+ size 1574355
data/delivers.parquet CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:48fd7da78b67a6f0d08d2280e96c328413767bf4eb5008c8e4f8621c9af961df
3
- size 421626330
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:18c5c1bdf59d4e6be3c9964cc33c00985b90596103011c3213a5c7967b3d067d
3
+ size 461745927
data/fpmmTrades.parquet CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:e0ad6be9794feae019e62609969208b27fea643f9784491f1e03e915883b6799
3
- size 4918148
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7b0025f18ea3f19eb7288b74a7ea0b131cb86d07ec62fe7d70c743d4fc6f4a07
3
+ size 4979104
data/fpmms.parquet CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d048fdde8de6ee49cfeb543b558007295fa260fb7ebdf23064df4d5f238a2b9f
3
- size 373210
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c333068cdd68f47d092633d9d47d80a11d3a58d9edc65b78750d199ba462bfe9
3
+ size 378079
data/invalid_trades.parquet CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:fb96a8b15d362395a0a7e243bff84b3b6354e265307b9a234c4fb508b67f7d7a
3
- size 219557
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dc913d782e4faea4d79636c858598aa157140f6739df7454eecfe37307f6d61f
3
+ size 284309
data/requests.parquet CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0cfc1cf7ec277e43cfa933e1298746fc337cc09f511c5162542ce876e3607fcb
3
- size 12785056
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:581d197f07231927fda68e37130d3fb25a08de5a40cb20136b6ad944b83e8551
3
+ size 12741805
data/summary_profitability.parquet CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:114c068eabf942a79389765a0a217b35c58c58811dc024d1b65e21f938ec7237
3
- size 33911
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0f401498cb4bd5a7d6974cd0e90f0454593cb621dc7d2d1c782d6153bb04a396
3
+ size 30517
data/t_map.pkl CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:42e0e362a115167660577f6504e2b92d9002cc837dbbe27d58ef2fb3b9dd14b2
3
- size 11623107
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2fd5ea8aef676670d7f3e0b9a63e16ba3568da98434880822be7846ba650b61b
3
+ size 12051980
data/tools.parquet CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:684f2b596ae39c06c36fbf3fe1eec93c5bb095861a6090b9cb5e917e29768337
3
- size 423650336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8e150b223fc99fe111b5b2b9aac841ce4cff6cbab3fbebd9a1b4397932258bad
3
+ size 465552676
data/tools_accuracy.csv CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:747914de5faf66fbefb889636fa9dd249adaca84bd0d40ae7462f1d66d395b56
3
  size 1008
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8ac964c50d037f954d2f3a733ec722883a29efa820aed537a77ce3b224640aa6
3
  size 1008
notebooks/weekly_analysis.ipynb CHANGED
@@ -9,7 +9,7 @@
9
  },
10
  {
11
  "cell_type": "code",
12
- "execution_count": 11,
13
  "metadata": {},
14
  "outputs": [],
15
  "source": [
@@ -22,7 +22,7 @@
22
  },
23
  {
24
  "cell_type": "code",
25
- "execution_count": 12,
26
  "metadata": {},
27
  "outputs": [],
28
  "source": [
@@ -51,7 +51,7 @@
51
  },
52
  {
53
  "cell_type": "code",
54
- "execution_count": 14,
55
  "metadata": {},
56
  "outputs": [],
57
  "source": [
@@ -511,229 +511,7 @@
511
  },
512
  {
513
  "cell_type": "code",
514
- "execution_count": 7,
515
- "metadata": {},
516
- "outputs": [
517
- {
518
- "data": {
519
- "text/html": [
520
- "<div>\n",
521
- "<style scoped>\n",
522
- " .dataframe tbody tr th:only-of-type {\n",
523
- " vertical-align: middle;\n",
524
- " }\n",
525
- "\n",
526
- " .dataframe tbody tr th {\n",
527
- " vertical-align: top;\n",
528
- " }\n",
529
- "\n",
530
- " .dataframe thead th {\n",
531
- " text-align: right;\n",
532
- " }\n",
533
- "</style>\n",
534
- "<table border=\"1\" class=\"dataframe\">\n",
535
- " <thead>\n",
536
- " <tr style=\"text-align: right;\">\n",
537
- " <th></th>\n",
538
- " <th>trader_address</th>\n",
539
- " <th>trade_id</th>\n",
540
- " <th>creation_timestamp</th>\n",
541
- " <th>title</th>\n",
542
- " <th>market_status</th>\n",
543
- " <th>collateral_amount</th>\n",
544
- " <th>outcome_index</th>\n",
545
- " <th>trade_fee_amount</th>\n",
546
- " <th>outcomes_tokens_traded</th>\n",
547
- " <th>current_answer</th>\n",
548
- " <th>is_invalid</th>\n",
549
- " <th>winning_trade</th>\n",
550
- " <th>earnings</th>\n",
551
- " <th>redeemed</th>\n",
552
- " <th>redeemed_amount</th>\n",
553
- " <th>num_mech_calls</th>\n",
554
- " <th>mech_fee_amount</th>\n",
555
- " <th>net_earnings</th>\n",
556
- " <th>roi</th>\n",
557
- " </tr>\n",
558
- " </thead>\n",
559
- " <tbody>\n",
560
- " <tr>\n",
561
- " <th>0</th>\n",
562
- " <td>0x022b36c50b85b8ae7addfb8a35d76c59d5814834</td>\n",
563
- " <td>0x017947579ab51313c31fe1cc562c0f1726ec09c90x02...</td>\n",
564
- " <td>2024-05-19 01:26:30+00:00</td>\n",
565
- " <td>Will Google's Pixel 9 lineup be officially rel...</td>\n",
566
- " <td>CLOSED</td>\n",
567
- " <td>0.638</td>\n",
568
- " <td>1</td>\n",
569
- " <td>0.013</td>\n",
570
- " <td>1.206</td>\n",
571
- " <td>1</td>\n",
572
- " <td>False</td>\n",
573
- " <td>True</td>\n",
574
- " <td>1.206</td>\n",
575
- " <td>True</td>\n",
576
- " <td>1.206</td>\n",
577
- " <td>0</td>\n",
578
- " <td>0.000</td>\n",
579
- " <td>0.556</td>\n",
580
- " <td>0.854</td>\n",
581
- " </tr>\n",
582
- " <tr>\n",
583
- " <th>1</th>\n",
584
- " <td>0x022b36c50b85b8ae7addfb8a35d76c59d5814834</td>\n",
585
- " <td>0x027f6bc849e273477f4a63085192714084917fcc0x02...</td>\n",
586
- " <td>2024-06-12 01:16:55+00:00</td>\n",
587
- " <td>Will the 2D-animated Paramount Plus show 'Tale...</td>\n",
588
- " <td>CLOSED</td>\n",
589
- " <td>1.000</td>\n",
590
- " <td>1</td>\n",
591
- " <td>0.020</td>\n",
592
- " <td>1.840</td>\n",
593
- " <td>1</td>\n",
594
- " <td>False</td>\n",
595
- " <td>True</td>\n",
596
- " <td>1.840</td>\n",
597
- " <td>True</td>\n",
598
- " <td>1.840</td>\n",
599
- " <td>5</td>\n",
600
- " <td>0.050</td>\n",
601
- " <td>0.770</td>\n",
602
- " <td>0.719</td>\n",
603
- " </tr>\n",
604
- " <tr>\n",
605
- " <th>2</th>\n",
606
- " <td>0x022b36c50b85b8ae7addfb8a35d76c59d5814834</td>\n",
607
- " <td>0x027f6bc849e273477f4a63085192714084917fcc0x02...</td>\n",
608
- " <td>2024-06-12 15:08:00+00:00</td>\n",
609
- " <td>Will the 2D-animated Paramount Plus show 'Tale...</td>\n",
610
- " <td>CLOSED</td>\n",
611
- " <td>0.800</td>\n",
612
- " <td>1</td>\n",
613
- " <td>0.016</td>\n",
614
- " <td>1.518</td>\n",
615
- " <td>1</td>\n",
616
- " <td>False</td>\n",
617
- " <td>True</td>\n",
618
- " <td>1.518</td>\n",
619
- " <td>True</td>\n",
620
- " <td>1.518</td>\n",
621
- " <td>5</td>\n",
622
- " <td>0.050</td>\n",
623
- " <td>0.652</td>\n",
624
- " <td>0.753</td>\n",
625
- " </tr>\n",
626
- " <tr>\n",
627
- " <th>3</th>\n",
628
- " <td>0x022b36c50b85b8ae7addfb8a35d76c59d5814834</td>\n",
629
- " <td>0x027f6bc849e273477f4a63085192714084917fcc0x02...</td>\n",
630
- " <td>2024-06-13 07:22:55+00:00</td>\n",
631
- " <td>Will the 2D-animated Paramount Plus show 'Tale...</td>\n",
632
- " <td>CLOSED</td>\n",
633
- " <td>0.456</td>\n",
634
- " <td>0</td>\n",
635
- " <td>0.009</td>\n",
636
- " <td>1.003</td>\n",
637
- " <td>1</td>\n",
638
- " <td>False</td>\n",
639
- " <td>False</td>\n",
640
- " <td>0.000</td>\n",
641
- " <td>True</td>\n",
642
- " <td>0.000</td>\n",
643
- " <td>5</td>\n",
644
- " <td>0.050</td>\n",
645
- " <td>-0.515</td>\n",
646
- " <td>-1.000</td>\n",
647
- " </tr>\n",
648
- " <tr>\n",
649
- " <th>4</th>\n",
650
- " <td>0x022b36c50b85b8ae7addfb8a35d76c59d5814834</td>\n",
651
- " <td>0x027f6bc849e273477f4a63085192714084917fcc0x02...</td>\n",
652
- " <td>2024-06-14 03:01:20+00:00</td>\n",
653
- " <td>Will the 2D-animated Paramount Plus show 'Tale...</td>\n",
654
- " <td>CLOSED</td>\n",
655
- " <td>0.704</td>\n",
656
- " <td>0</td>\n",
657
- " <td>0.014</td>\n",
658
- " <td>1.198</td>\n",
659
- " <td>1</td>\n",
660
- " <td>False</td>\n",
661
- " <td>False</td>\n",
662
- " <td>0.000</td>\n",
663
- " <td>True</td>\n",
664
- " <td>0.000</td>\n",
665
- " <td>5</td>\n",
666
- " <td>0.050</td>\n",
667
- " <td>-0.768</td>\n",
668
- " <td>-1.000</td>\n",
669
- " </tr>\n",
670
- " </tbody>\n",
671
- "</table>\n",
672
- "</div>"
673
- ],
674
- "text/plain": [
675
- " trader_address \\\n",
676
- "0 0x022b36c50b85b8ae7addfb8a35d76c59d5814834 \n",
677
- "1 0x022b36c50b85b8ae7addfb8a35d76c59d5814834 \n",
678
- "2 0x022b36c50b85b8ae7addfb8a35d76c59d5814834 \n",
679
- "3 0x022b36c50b85b8ae7addfb8a35d76c59d5814834 \n",
680
- "4 0x022b36c50b85b8ae7addfb8a35d76c59d5814834 \n",
681
- "\n",
682
- " trade_id \\\n",
683
- "0 0x017947579ab51313c31fe1cc562c0f1726ec09c90x02... \n",
684
- "1 0x027f6bc849e273477f4a63085192714084917fcc0x02... \n",
685
- "2 0x027f6bc849e273477f4a63085192714084917fcc0x02... \n",
686
- "3 0x027f6bc849e273477f4a63085192714084917fcc0x02... \n",
687
- "4 0x027f6bc849e273477f4a63085192714084917fcc0x02... \n",
688
- "\n",
689
- " creation_timestamp \\\n",
690
- "0 2024-05-19 01:26:30+00:00 \n",
691
- "1 2024-06-12 01:16:55+00:00 \n",
692
- "2 2024-06-12 15:08:00+00:00 \n",
693
- "3 2024-06-13 07:22:55+00:00 \n",
694
- "4 2024-06-14 03:01:20+00:00 \n",
695
- "\n",
696
- " title market_status \\\n",
697
- "0 Will Google's Pixel 9 lineup be officially rel... CLOSED \n",
698
- "1 Will the 2D-animated Paramount Plus show 'Tale... CLOSED \n",
699
- "2 Will the 2D-animated Paramount Plus show 'Tale... CLOSED \n",
700
- "3 Will the 2D-animated Paramount Plus show 'Tale... CLOSED \n",
701
- "4 Will the 2D-animated Paramount Plus show 'Tale... CLOSED \n",
702
- "\n",
703
- " collateral_amount outcome_index trade_fee_amount outcomes_tokens_traded \\\n",
704
- "0 0.638 1 0.013 1.206 \n",
705
- "1 1.000 1 0.020 1.840 \n",
706
- "2 0.800 1 0.016 1.518 \n",
707
- "3 0.456 0 0.009 1.003 \n",
708
- "4 0.704 0 0.014 1.198 \n",
709
- "\n",
710
- " current_answer is_invalid winning_trade earnings redeemed \\\n",
711
- "0 1 False True 1.206 True \n",
712
- "1 1 False True 1.840 True \n",
713
- "2 1 False True 1.518 True \n",
714
- "3 1 False False 0.000 True \n",
715
- "4 1 False False 0.000 True \n",
716
- "\n",
717
- " redeemed_amount num_mech_calls mech_fee_amount net_earnings roi \n",
718
- "0 1.206 0 0.000 0.556 0.854 \n",
719
- "1 1.840 5 0.050 0.770 0.719 \n",
720
- "2 1.518 5 0.050 0.652 0.753 \n",
721
- "3 0.000 5 0.050 -0.515 -1.000 \n",
722
- "4 0.000 5 0.050 -0.768 -1.000 "
723
- ]
724
- },
725
- "execution_count": 7,
726
- "metadata": {},
727
- "output_type": "execute_result"
728
- }
729
- ],
730
- "source": [
731
- "all_trades.head()"
732
- ]
733
- },
734
- {
735
- "cell_type": "code",
736
- "execution_count": 17,
737
  "metadata": {},
738
  "outputs": [
739
  {
@@ -741,31 +519,31 @@
741
  "output_type": "stream",
742
  "text": [
743
  "<class 'pandas.core.frame.DataFrame'>\n",
744
- "RangeIndex: 18941 entries, 0 to 18940\n",
745
  "Data columns (total 19 columns):\n",
746
  " # Column Non-Null Count Dtype \n",
747
  "--- ------ -------------- ----- \n",
748
- " 0 trader_address 18941 non-null object \n",
749
- " 1 trade_id 18941 non-null object \n",
750
- " 2 creation_timestamp 18941 non-null datetime64[ns, UTC]\n",
751
- " 3 title 18941 non-null object \n",
752
- " 4 market_status 18941 non-null object \n",
753
- " 5 collateral_amount 18941 non-null float64 \n",
754
- " 6 outcome_index 18941 non-null object \n",
755
- " 7 trade_fee_amount 18941 non-null float64 \n",
756
- " 8 outcomes_tokens_traded 18941 non-null float64 \n",
757
- " 9 current_answer 18941 non-null int64 \n",
758
- " 10 is_invalid 18941 non-null bool \n",
759
- " 11 winning_trade 18941 non-null bool \n",
760
- " 12 earnings 18941 non-null float64 \n",
761
- " 13 redeemed 18941 non-null bool \n",
762
- " 14 redeemed_amount 18941 non-null float64 \n",
763
- " 15 num_mech_calls 18941 non-null int64 \n",
764
- " 16 mech_fee_amount 18941 non-null float64 \n",
765
- " 17 net_earnings 18941 non-null float64 \n",
766
- " 18 roi 18941 non-null float64 \n",
767
  "dtypes: bool(3), datetime64[ns, UTC](1), float64(8), int64(2), object(5)\n",
768
- "memory usage: 2.4+ MB\n"
769
  ]
770
  }
771
  ],
 
9
  },
10
  {
11
  "cell_type": "code",
12
+ "execution_count": 1,
13
  "metadata": {},
14
  "outputs": [],
15
  "source": [
 
22
  },
23
  {
24
  "cell_type": "code",
25
+ "execution_count": 2,
26
  "metadata": {},
27
  "outputs": [],
28
  "source": [
 
51
  },
52
  {
53
  "cell_type": "code",
54
+ "execution_count": 3,
55
  "metadata": {},
56
  "outputs": [],
57
  "source": [
 
511
  },
512
  {
513
  "cell_type": "code",
514
+ "execution_count": 4,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
515
  "metadata": {},
516
  "outputs": [
517
  {
 
519
  "output_type": "stream",
520
  "text": [
521
  "<class 'pandas.core.frame.DataFrame'>\n",
522
+ "RangeIndex: 17067 entries, 0 to 17066\n",
523
  "Data columns (total 19 columns):\n",
524
  " # Column Non-Null Count Dtype \n",
525
  "--- ------ -------------- ----- \n",
526
+ " 0 trader_address 17067 non-null object \n",
527
+ " 1 trade_id 17067 non-null object \n",
528
+ " 2 creation_timestamp 17067 non-null datetime64[ns, UTC]\n",
529
+ " 3 title 17067 non-null object \n",
530
+ " 4 market_status 17067 non-null object \n",
531
+ " 5 collateral_amount 17067 non-null float64 \n",
532
+ " 6 outcome_index 17067 non-null object \n",
533
+ " 7 trade_fee_amount 17067 non-null float64 \n",
534
+ " 8 outcomes_tokens_traded 17067 non-null float64 \n",
535
+ " 9 current_answer 17067 non-null int64 \n",
536
+ " 10 is_invalid 17067 non-null bool \n",
537
+ " 11 winning_trade 17067 non-null bool \n",
538
+ " 12 earnings 17067 non-null float64 \n",
539
+ " 13 redeemed 17067 non-null bool \n",
540
+ " 14 redeemed_amount 17067 non-null float64 \n",
541
+ " 15 num_mech_calls 17067 non-null int64 \n",
542
+ " 16 mech_fee_amount 17067 non-null float64 \n",
543
+ " 17 net_earnings 17067 non-null float64 \n",
544
+ " 18 roi 17067 non-null float64 \n",
545
  "dtypes: bool(3), datetime64[ns, UTC](1), float64(8), int64(2), object(5)\n",
546
+ "memory usage: 2.1+ MB\n"
547
  ]
548
  }
549
  ],
scripts/pull_data.py CHANGED
@@ -18,7 +18,7 @@ from tools import (
18
  update_tools_accuracy,
19
  )
20
  from profitability import run_profitability_analysis
21
- from utils import get_question, current_answer
22
  from get_mech_info import get_mech_info_last_60_days
23
  from update_tools_accuracy import compute_tools_accuracy
24
  import gc
@@ -47,8 +47,7 @@ def parallelize_timestamp_conversion(df: pd.DataFrame, function: callable) -> li
47
  return results
48
 
49
 
50
- def updating_timestamps(rpc: str):
51
- web3 = Web3(Web3.HTTPProvider(rpc))
52
  # Get currentAnswer from FPMMS
53
  fpmms = pd.read_parquet(DATA_DIR / MARKETS_FILENAME)
54
  tools = pd.read_parquet(DATA_DIR / TOOLS_FILENAME)
@@ -60,12 +59,24 @@ def updating_timestamps(rpc: str):
60
 
61
  tools["currentAnswer"] = tools["currentAnswer"].str.replace("yes", "Yes")
62
  tools["currentAnswer"] = tools["currentAnswer"].str.replace("no", "No")
 
 
 
 
 
 
 
 
 
63
 
64
  # Convert block number to timestamp
65
  logging.info("Converting block number to timestamp")
66
  t_map = pickle.load(open(DATA_DIR / "t_map.pkl", "rb"))
67
  tools["request_time"] = tools["request_block"].map(t_map)
68
 
 
 
 
69
  # Identify tools with missing request_time and fill them
70
  missing_time_indices = tools[tools["request_time"].isna()].index
71
  if not missing_time_indices.empty:
@@ -104,14 +115,13 @@ def updating_timestamps(rpc: str):
104
 
105
  # clean and release all memory
106
  del tools
107
- del fpmms
108
  del t_map
109
  gc.collect()
110
 
111
 
112
  def weekly_analysis():
113
  """Run weekly analysis for the FPMMS project."""
114
- rpc = "https://lb.nodies.app/v1/406d8dcc043f4cb3959ed7d6673d311a"
115
 
116
  # Run markets ETL
117
  logging.info("Running markets ETL")
@@ -137,8 +147,12 @@ def weekly_analysis():
137
  rpc=rpc,
138
  )
139
  logging.info("Profitability analysis completed")
140
-
141
- updating_timestamps(rpc)
 
 
 
 
142
 
143
  compute_tools_accuracy()
144
 
@@ -147,5 +161,5 @@ def weekly_analysis():
147
 
148
  if __name__ == "__main__":
149
  weekly_analysis()
150
- # rpc = "https://lb.nodies.app/v1/406d8dcc043f4cb3959ed7d6673d311a"
151
  # updating_timestamps(rpc)
 
18
  update_tools_accuracy,
19
  )
20
  from profitability import run_profitability_analysis
21
+ from utils import get_question, current_answer, RPC
22
  from get_mech_info import get_mech_info_last_60_days
23
  from update_tools_accuracy import compute_tools_accuracy
24
  import gc
 
47
  return results
48
 
49
 
50
+ def add_current_answer():
 
51
  # Get currentAnswer from FPMMS
52
  fpmms = pd.read_parquet(DATA_DIR / MARKETS_FILENAME)
53
  tools = pd.read_parquet(DATA_DIR / TOOLS_FILENAME)
 
59
 
60
  tools["currentAnswer"] = tools["currentAnswer"].str.replace("yes", "Yes")
61
  tools["currentAnswer"] = tools["currentAnswer"].str.replace("no", "No")
62
+ # Save the tools data after the updates on the content
63
+ tools.to_parquet(DATA_DIR / TOOLS_FILENAME, index=False)
64
+ del fpmms
65
+
66
+
67
+ def updating_timestamps(rpc: str):
68
+ web3 = Web3(Web3.HTTPProvider(rpc))
69
+
70
+ tools = pd.read_parquet(DATA_DIR / TOOLS_FILENAME)
71
 
72
  # Convert block number to timestamp
73
  logging.info("Converting block number to timestamp")
74
  t_map = pickle.load(open(DATA_DIR / "t_map.pkl", "rb"))
75
  tools["request_time"] = tools["request_block"].map(t_map)
76
 
77
+ no_data = tools["request_time"].isna().sum()
78
+ logging.info(f"Total rows with no request time info = {no_data}")
79
+
80
  # Identify tools with missing request_time and fill them
81
  missing_time_indices = tools[tools["request_time"].isna()].index
82
  if not missing_time_indices.empty:
 
115
 
116
  # clean and release all memory
117
  del tools
 
118
  del t_map
119
  gc.collect()
120
 
121
 
122
  def weekly_analysis():
123
  """Run weekly analysis for the FPMMS project."""
124
+ rpc = RPC
125
 
126
  # Run markets ETL
127
  logging.info("Running markets ETL")
 
147
  rpc=rpc,
148
  )
149
  logging.info("Profitability analysis completed")
150
+ add_current_answer()
151
+ try:
152
+ updating_timestamps(rpc)
153
+ except Exception as e:
154
+ logging.error("Error while updating timestamps of tools")
155
+ print(e)
156
 
157
  compute_tools_accuracy()
158
 
 
161
 
162
  if __name__ == "__main__":
163
  weekly_analysis()
164
+ # rpc = RPC
165
  # updating_timestamps(rpc)
scripts/tools.py CHANGED
@@ -166,20 +166,25 @@ def get_events(
166
  )
167
  break
168
  sleep = SLEEP * retries
169
- if (
170
- (
171
- isinstance(exc, ValueError)
172
- and re.match(
173
- RE_RPC_FILTER_ERROR, exc.args[0].get("message", "")
174
- )
175
- is None
176
- )
177
- and not isinstance(exc, ValueError)
178
- and not isinstance(exc, MismatchedABI)
179
- ):
180
- tqdm.write(
181
- f"An error was raised from the RPC: {exc}\n Retrying in {sleep} seconds."
182
- )
 
 
 
 
 
183
  time.sleep(sleep)
184
 
185
  from_block += batch_size
@@ -514,11 +519,16 @@ def update_tools_accuracy(
514
 
515
  print("Wins dataset")
516
  print(wins.head())
517
- timeline = tools_non_error.groupby(["tool"])["request_time"].agg(["min", "max"])
518
-
519
- print("timeline dataset")
520
- print(timeline.head())
521
- acc_info = wins.merge(timeline, how="left", on="tool")
 
 
 
 
 
522
 
523
  if tools_acc is None:
524
  print("Creating accuracy file for the first time")
@@ -534,8 +544,12 @@ def update_tools_accuracy(
534
  if tool in existing_tools:
535
  new_accuracy = acc_info[acc_info["tool"] == tool]["tool_accuracy"].values[0]
536
  new_volume = acc_info[acc_info["tool"] == tool]["total_requests"].values[0]
537
- new_min_timeline = acc_info[acc_info["tool"] == tool]["min"].values[0]
538
- new_max_timeline = acc_info[acc_info["tool"] == tool]["max"].values[0]
 
 
 
 
539
  tools_acc.loc[tools_acc["tool"] == tool, "tool_accuracy"] = new_accuracy
540
  tools_acc.loc[tools_acc["tool"] == tool, "total_requests"] = new_volume
541
  tools_acc.loc[tools_acc["tool"] == tool, "min"] = new_min_timeline
 
166
  )
167
  break
168
  sleep = SLEEP * retries
169
+ # error_message = ""
170
+ # if isinstance(exc.args[0], str):
171
+ # error_message = exc.args[0]
172
+ # elif isinstance(exc, ValueError):
173
+ # error_message = exc.args[0].get("message", "")
174
+ # if (
175
+ # (
176
+ # isinstance(exc, ValueError)
177
+ # and re.match(RE_RPC_FILTER_ERROR, error_message) is None
178
+ # )
179
+ # and not isinstance(exc, ValueError)
180
+ # and not isinstance(exc, MismatchedABI)
181
+ # ):
182
+
183
+ tqdm.write(
184
+ f"An error was raised from the RPC: {exc}\n Retrying in {sleep} seconds."
185
+ )
186
+ if hasattr(exc, "message"):
187
+ tqdm.write(f"Error message: {exc.messge}\n")
188
  time.sleep(sleep)
189
 
190
  from_block += batch_size
 
519
 
520
  print("Wins dataset")
521
  print(wins.head())
522
+ no_timeline_info = False
523
+ try:
524
+ timeline = tools_non_error.groupby(["tool"])["request_time"].agg(["min", "max"])
525
+ print("timeline dataset")
526
+ print(timeline.head())
527
+ acc_info = wins.merge(timeline, how="left", on="tool")
528
+ except:
529
+ print("NO REQUEST TIME INFORMATION AVAILABLE")
530
+ no_timeline_info = True
531
+ acc_info = wins
532
 
533
  if tools_acc is None:
534
  print("Creating accuracy file for the first time")
 
544
  if tool in existing_tools:
545
  new_accuracy = acc_info[acc_info["tool"] == tool]["tool_accuracy"].values[0]
546
  new_volume = acc_info[acc_info["tool"] == tool]["total_requests"].values[0]
547
+ if no_timeline_info:
548
+ new_min_timeline = None
549
+ new_max_timeline = None
550
+ else:
551
+ new_min_timeline = acc_info[acc_info["tool"] == tool]["min"].values[0]
552
+ new_max_timeline = acc_info[acc_info["tool"] == tool]["max"].values[0]
553
  tools_acc.loc[tools_acc["tool"] == tool, "tool_accuracy"] = new_accuracy
554
  tools_acc.loc[tools_acc["tool"] == tool, "total_requests"] = new_volume
555
  tools_acc.loc[tools_acc["tool"] == tool, "min"] = new_min_timeline
scripts/utils.py CHANGED
@@ -47,6 +47,7 @@ INC_TOOLS = [
47
  ]
48
 
49
  SUBGRAPH_API_KEY = os.environ.get("SUBGRAPH_API_KEY", None)
 
50
 
51
 
52
  class MechEventName(Enum):
 
47
  ]
48
 
49
  SUBGRAPH_API_KEY = os.environ.get("SUBGRAPH_API_KEY", None)
50
+ RPC = os.environ.get("RPC", None)
51
 
52
 
53
  class MechEventName(Enum):