Update main.py

main.py

@@ -205,48 +205,7 @@ previous_content =  P("""The performance of a large language model (LLM)
                     and the process followed to create its 📚
                     FineWeb-Edu subset.""")
 
-
-
-
-
-
-
-
-@app.get("/intro")
-def intro():
-    return Div(
-        Section(
-            H2("Introduction"),
-            intro_text,
-            intro_list,
-            intro_1,
-            intro_2,
-            intro_3,
-            intro_4,
-            intro_5,
-            id="section1",
-        ),
-        Section(
-            H3("Global Deduplication"),
-            P("TxT360 curated a wide range of datasets, including a whopping 99 Common Crawl Dumps and  a list of high quality datasets: StackExchange, Wikipedia, Arxiv, USPTO, DM Math, HackerNews, Ubuntu IRC, Europarl, FreeLaw, PG19, S2ORC, PhilPapers, PubMed Abstracts, and PubMed Central. For the first time in a released dataset, we locally and globally deduplicated the data across each dataset creating the highest quality data available."),
-            id="section2",
-        ),
-        Section(
-            H3("Controllable Upweighting for Flexible Data Sample Weight Control"),
-            P("In large-scale corpora like CommonCrawl, text duplication is a frequent occurrence. Duplication can be considered as a natural upsampling of some data points. Recent studies have highlighted the potential drawbacks of oversampling specific data points, which can negatively impact pretraining performance [2205.10487]. However, when samples are repeated appropriately, the performance can actually improve [2306.01116, 2305.16264, 2406.11794, FineWeb]. Despite this, there is currently no widely accepted best practice for data sampling, and it’s unlikely that a one-size-fits-all approach will emerge given the scale of these datasets. Previous work either leaves the deduplication process to the user (as seen in RedPajama V2 and DCLM-Pool) or provides a corpus that has been downsampled in a specific manner (such as in FineWeb and RefinedWeb)."),
-            P("Given the high cost of deduplication, TxT360 offers a complete deduplication across all datasets (so you don’t have to). Additionally, TxT360 maintains detailed metadata for each sample, including the frequency and location of duplicates. This metadata gives pretrainers the flexibility to adjust the weight of samples as needed. In principle, one can recover the original dataset distribution (footnote: this approach also means a smaller size on disk). We will demonstrate a simple upsampling strategy that results in an effective pretraining dataset. "),
-            id="section3",
-        ),
-        Section(
-            H3("Full and Openly Documented Production Ready Pretraining Corpus"),
-            P("We cover every aspect of the decisions made to produce the dataset, including document selection, filtering, quality assurance, deduplication, standardization and PII.  Our reasoning is thoroughly explained, ensuring transparency and replicability. "),
-            P("Our code is open sourced here[link to github]."),
-            P("The dataset is ready for immediate download directly from Hugging Face [link]."),
-            P("In the remainder of this blog post, we will walk you through the entire process and the rationale behind each decision. Enjoy!"),
-            id="section4",
-        ),
-
-    dataset_comparison = pd.DataFrame(
+dataset_comparison = pd.DataFrame(
         {
             "Dataset": [
                 "TxT360",
@@ -383,10 +342,45 @@ def intro():
 
     table_html = preprocessing_steps.to_html(index=False, border=0)
     table_div = Div(NotStr(table_html), style="margin: 40px;")
-    
 
 
-        
+
+
+
+
+@app.get("/intro")
+def intro():
+    return Div(
+        Section(
+            H2("Introduction"),
+            intro_text,
+            intro_list,
+            intro_1,
+            intro_2,
+            intro_3,
+            intro_4,
+            intro_5,
+            id="section1",
+        ),
+        Section(
+            H3("Global Deduplication"),
+            P("TxT360 curated a wide range of datasets, including a whopping 99 Common Crawl Dumps and  a list of high quality datasets: StackExchange, Wikipedia, Arxiv, USPTO, DM Math, HackerNews, Ubuntu IRC, Europarl, FreeLaw, PG19, S2ORC, PhilPapers, PubMed Abstracts, and PubMed Central. For the first time in a released dataset, we locally and globally deduplicated the data across each dataset creating the highest quality data available."),
+            id="section2",
+        ),
+        Section(
+            H3("Controllable Upweighting for Flexible Data Sample Weight Control"),
+            P("In large-scale corpora like CommonCrawl, text duplication is a frequent occurrence. Duplication can be considered as a natural upsampling of some data points. Recent studies have highlighted the potential drawbacks of oversampling specific data points, which can negatively impact pretraining performance [2205.10487]. However, when samples are repeated appropriately, the performance can actually improve [2306.01116, 2305.16264, 2406.11794, FineWeb]. Despite this, there is currently no widely accepted best practice for data sampling, and it’s unlikely that a one-size-fits-all approach will emerge given the scale of these datasets. Previous work either leaves the deduplication process to the user (as seen in RedPajama V2 and DCLM-Pool) or provides a corpus that has been downsampled in a specific manner (such as in FineWeb and RefinedWeb)."),
+            P("Given the high cost of deduplication, TxT360 offers a complete deduplication across all datasets (so you don’t have to). Additionally, TxT360 maintains detailed metadata for each sample, including the frequency and location of duplicates. This metadata gives pretrainers the flexibility to adjust the weight of samples as needed. In principle, one can recover the original dataset distribution (footnote: this approach also means a smaller size on disk). We will demonstrate a simple upsampling strategy that results in an effective pretraining dataset. "),
+            id="section3",
+        ),
+        Section(
+            H3("Full and Openly Documented Production Ready Pretraining Corpus"),
+            P("We cover every aspect of the decisions made to produce the dataset, including document selection, filtering, quality assurance, deduplication, standardization and PII.  Our reasoning is thoroughly explained, ensuring transparency and replicability. "),
+            P("Our code is open sourced here[link to github]."),
+            P("The dataset is ready for immediate download directly from Hugging Face [link]."),
+            P("In the remainder of this blog post, we will walk you through the entire process and the rationale behind each decision. Enjoy!"),
+            id="section4",
+        ),
         Section(
             H2("Combining the Best of Web and Curated Sources"),
             H3("Why combine the web and highly curated sources? Isn't the web-only data enough?"),