directly use file

index.html

@@ -237,52 +237,68 @@
       async function fetchTotalAnnotations() {
         try {
           const stats = { total: 0, languages: new Set(), users: new Set() };
-          let offset = 0;
-          const limit = 100; // API default limit
-          let hasMore = true;
 
           document.getElementById("total-annotations").innerHTML = `
             <div class="loading"></div>
-            <p>Processing rows...</p>
+            <p>Loading data...</p>
           `;
 
-          while (hasMore) {
-            const response = await fetch(
-              `https://datasets-server.huggingface.co/rows?dataset=data-is-better-together%2Ffineweb-c-progress&config=default&split=train&offset=${offset}`
-            );
-
-            if (!response.ok) {
-              throw new Error(`HTTP error! status: ${response.status}`);
-            }
-
-            const data = await response.json();
-
-            // Process this batch of rows
-            data.rows.forEach((row) => {
-              const submissions = parseInt(row.row.submitted) || 0;
-              stats.total += submissions;
-              if (submissions > 0) {
-                stats.languages.add(row.row.language_dataset_name);
-                if (row.row.username) {
-                  stats.users.add(row.row.username);
+          const response = await fetch(
+            "https://huggingface.co/datasets/data-is-better-together/fineweb-c-progress/resolve/main/argilla_progress.ndjson"
+          );
+          if (!response.ok)
+            throw new Error(`HTTP error! status: ${response.status}`);
+
+          const reader = response.body.getReader();
+          const decoder = new TextDecoder();
+          let buffer = "";
+
+          while (true) {
+            const { value, done } = await reader.read();
+            if (done) break;
+
+            buffer += decoder.decode(value, { stream: true });
+            const lines = buffer.split("\n");
+            buffer = lines.pop() || ""; // Keep the last incomplete line in buffer
+
+            for (const line of lines) {
+              if (!line.trim()) continue;
+              try {
+                const row = JSON.parse(line);
+                const submissions = parseInt(row.submitted) || 0;
+                stats.total += submissions;
+                if (submissions > 0) {
+                  stats.languages.add(row.language_dataset_name);
+                  if (row.username) {
+                    stats.users.add(row.username);
+                  }
                 }
+              } catch (e) {
+                console.error("Error parsing line:", e);
               }
-            });
+            }
 
-            // Update progress
+            // Update progress periodically
             document.getElementById("total-annotations").innerHTML = `
               <div class="loading"></div>
               <p>Processing... Current count: <strong>${stats.total.toLocaleString()}</strong></p>
-              <p>Processed ${offset + data.rows.length} rows</p>
             `;
+          }
 
-            // Check if we should continue
-            if (data.rows.length < limit) {
-              hasMore = false;
-            } else {
-              offset += limit;
-              // Add a small delay to avoid overwhelming the API
-              await new Promise((resolve) => setTimeout(resolve, 100));
+          // Process any remaining data in buffer
+          if (buffer.trim()) {
+            try {
+              const row = JSON.parse(buffer);
+              const submissions = parseInt(row.submitted) || 0;
+              stats.total += submissions;
+              if (submissions > 0) {
+                stats.languages.add(row.language_dataset_name);
+                if (row.username) {
+                  stats.users.add(row.username);
+                }
+              }
+            } catch (e) {
+              console.error("Error parsing final line:", e);
             }
           }