fineweb-c-dashboard

Running

App Files Files Community

dvilasuero HF staff commited on Dec 10, 2024

Commit

c5c6da9

verified ·

1 Parent(s): 208151a

Update index.html

Browse files

Files changed (1) hide show

index.html +44 -20

index.html CHANGED Viewed

@@ -164,7 +164,7 @@
   <body>
     <div class="container">
       <header class="header">
-        <h1>🏆 🌐 FineWeb 2 Sprint Leaderboard</h1>
       </header>
       <div class="summary" id="total-annotations">
@@ -202,33 +202,57 @@
     <script>
       async function fetchTotalAnnotations() {
         try {
-          const response = await fetch(
-            "https://datasets-server.huggingface.co/rows?dataset=data-is-better-together/fineweb-c-progress&config=default&split=train&offset=0&length=100"
-          );
-          if (!response.ok) {
-            throw new Error(`HTTP error! status: ${response.status}`);
-          }
-          const data = await response.json();
-          // Calculate total annotations and count languages with submissions
-          const stats = data.rows.reduce(
-            (acc, row) => {
-              const submissions = row.row.submitted || 0;
               if (submissions > 0) {
-                acc.languages.add(row.row.language_dataset_name);
-                // Add username to the Set if it exists
                 if (row.row.username) {
-                  acc.users.add(row.row.username);
                 }
               }
-              acc.total += submissions;
-              return acc;
-            },
-            { total: 0, languages: new Set(), users: new Set() }
-          );
           document.getElementById("total-annotations").innerHTML = `
             <p>Total annotations submitted: <strong>${stats.total.toLocaleString()}</strong></p>
             <p>Languages with annotations: <strong>${stats.languages.size}</strong></p>

   <body>
     <div class="container">
       <header class="header">
+        <h1>🏆 FineWeb 2 Sprint Leaderboard</h1>
       </header>
       <div class="summary" id="total-annotations">
     <script>
       async function fetchTotalAnnotations() {
         try {
+          const stats = { total: 0, languages: new Set(), users: new Set() };
+          let offset = 0;
+          const limit = 100; // API default limit
+          let hasMore = true;
+          document.getElementById("total-annotations").innerHTML = `
+            <div class="loading"></div>
+            <p>Processing rows...</p>
+          `;
+          while (hasMore) {
+            const response = await fetch(
+              `https://datasets-server.huggingface.co/rows?dataset=data-is-better-together%2Ffineweb-c-progress&config=default&split=train&offset=${offset}`
+            );
+            if (!response.ok) {
+              throw new Error(`HTTP error! status: ${response.status}`);
+            }
+            const data = await response.json();
+            // Process this batch of rows
+            data.rows.forEach(row => {
+              const submissions = parseInt(row.row.submitted) || 0;
+              stats.total += submissions;
               if (submissions > 0) {
+                stats.languages.add(row.row.language_dataset_name);
                 if (row.row.username) {
+                  stats.users.add(row.row.username);
                 }
               }
+            });
+            // Update progress
+            document.getElementById("total-annotations").innerHTML = `
+              <div class="loading"></div>
+              <p>Processing... Current count: <strong>${stats.total.toLocaleString()}</strong></p>
+              <p>Processed ${offset + data.rows.length} rows</p>
+            `;
+            // Check if we should continue
+            if (data.rows.length < limit) {
+              hasMore = false;
+            } else {
+              offset += limit;
+              // Add a small delay to avoid overwhelming the API
+              await new Promise(resolve => setTimeout(resolve, 100));
+            }
+          }
+          // Final update
           document.getElementById("total-annotations").innerHTML = `
             <p>Total annotations submitted: <strong>${stats.total.toLocaleString()}</strong></p>
             <p>Languages with annotations: <strong>${stats.languages.size}</strong></p>