Push to HF space

Dockerfile

@@ -25,14 +25,15 @@ RUN python -m pip install lilac[all]
 COPY --chown=user /dist ./dist/
 RUN python -m pip install --find-links=dist --upgrade lilac[all]
 
-COPY --chown=user .env .
-COPY --chown=user .env.demo .
+# Install the huggingface hub, used to download files.
+RUN pip install huggingface_hub
+
 # Copy the README so we can read the datasets from the HuggingFace config.
 COPY --chown=user README.md .
 # Copy the license just in case.
 COPY --chown=user LICENSE .
 
-COPY --chown=user docker_start.sh docker_start.py ./
+COPY --chown=user docker_start.sh ./
 
 # Make a local data directory for non-persistent storage demos.
 RUN mkdir -p ./data

README.md

@@ -6,13 +6,11 @@ colorTo: purple
 sdk: docker
 app_port: 5432
 datasets:
+  - lilacai/lilac-textbook_quality_programming
   - lilacai/lilac-piqa
   - lilacai/lilac-science-qa-derek-thomas
   - lilacai/lilac-enron-emails
-  - lilacai/lilac-mmlu_professional_law
-  - lilacai/lilac-pile-of-law-r-legaladvice
   - lilacai/lilac-open-asssistant-conversations
-  - lilacai/lilac-squad_v2
   - lilacai/lilac-imdb
   - lilacai/lilac-opus100-en-es-validation
   - lilacai/lilac-databricks-dolly-15k-curated-en

data/lilac.yml

@@ -279,277 +279,6 @@ datasets:
           - text
         markdown_paths: []
       preferred_embedding: gte-small
-  - namespace: lilac
-    name: squad_v2
-    tags: []
-    source:
-      dataset_name: squad_v2
-      config_name: null
-      split: null
-      sample_size: null
-      revision: null
-      load_from_disk: false
-      source_name: huggingface
-    embeddings:
-      - path: context
-        embedding: gte-small
-    signals:
-      - path: context
-        signal:
-          threshold: 0.85
-          signal_name: near_dup
-      - path: context
-        signal:
-          signal_name: pii
-      - path: context
-        signal:
-          split_by_paragraph: false
-          signal_name: lang_detection
-      - path: context
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: positive-sentiment
-          draft: main
-          signal_name: concept_score
-      - path: context
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: non-english
-          draft: main
-          signal_name: concept_score
-      - path: context
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: toxicity
-          draft: main
-          signal_name: concept_score
-      - path: context
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: question
-          draft: main
-          signal_name: concept_score
-      - path: context
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: legal-termination
-          draft: main
-          signal_name: concept_score
-      - path: context
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: source-code
-          draft: main
-          signal_name: concept_score
-      - path: context
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: negative-sentiment
-          draft: main
-          signal_name: concept_score
-      - path: context
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: profanity
-          draft: main
-          signal_name: concept_score
-      - path: context
-        signal:
-          signal_name: text_statistics
-      - path: question
-        signal:
-          threshold: 0.85
-          signal_name: near_dup
-      - path: question
-        signal:
-          signal_name: pii
-      - path: question
-        signal:
-          split_by_paragraph: false
-          signal_name: lang_detection
-      - path: question
-        signal:
-          signal_name: text_statistics
-      - path:
-          - answers
-          - text
-          - '*'
-        signal:
-          threshold: 0.85
-          signal_name: near_dup
-      - path:
-          - answers
-          - text
-          - '*'
-        signal:
-          signal_name: pii
-      - path:
-          - answers
-          - text
-          - '*'
-        signal:
-          split_by_paragraph: false
-          signal_name: lang_detection
-      - path:
-          - answers
-          - text
-          - '*'
-        signal:
-          signal_name: text_statistics
-      - path: question
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: legal-termination
-          draft: main
-          signal_name: concept_score
-      - path: question
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: negative-sentiment
-          draft: main
-          signal_name: concept_score
-      - path: question
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: non-english
-          draft: main
-          signal_name: concept_score
-      - path: question
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: positive-sentiment
-          draft: main
-          signal_name: concept_score
-      - path: question
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: profanity
-          draft: main
-          signal_name: concept_score
-      - path: question
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: question
-          draft: main
-          signal_name: concept_score
-      - path: question
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: source-code
-          draft: main
-          signal_name: concept_score
-      - path: question
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: toxicity
-          draft: main
-          signal_name: concept_score
-      - path:
-          - answers
-          - text
-          - '*'
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: legal-termination
-          draft: main
-          signal_name: concept_score
-      - path:
-          - answers
-          - text
-          - '*'
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: negative-sentiment
-          draft: main
-          signal_name: concept_score
-      - path:
-          - answers
-          - text
-          - '*'
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: non-english
-          draft: main
-          signal_name: concept_score
-      - path:
-          - answers
-          - text
-          - '*'
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: positive-sentiment
-          draft: main
-          signal_name: concept_score
-      - path:
-          - answers
-          - text
-          - '*'
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: profanity
-          draft: main
-          signal_name: concept_score
-      - path:
-          - answers
-          - text
-          - '*'
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: question
-          draft: main
-          signal_name: concept_score
-      - path:
-          - answers
-          - text
-          - '*'
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: source-code
-          draft: main
-          signal_name: concept_score
-      - path:
-          - answers
-          - text
-          - '*'
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: toxicity
-          draft: main
-          signal_name: concept_score
-    settings:
-      ui:
-        media_paths:
-          - context
-          - question
-          - - answers
-            - text
-            - '*'
-        markdown_paths: []
-      preferred_embedding: gte-small
   - namespace: lilac
     name: databricks-dolly-15k-curated-en
     tags: []
@@ -1735,319 +1464,28 @@ datasets:
         markdown_paths: []
       preferred_embedding: gte-small
   - namespace: lilac
-    name: mmlu_professional_law
+    name: science-qa-derek-thomas
     tags: []
     source:
-      dataset_name: cais/mmlu
-      config_name: professional_law
+      dataset_name: derek-thomas/ScienceQA
+      config_name: null
       split: null
       sample_size: null
       revision: null
       load_from_disk: false
       source_name: huggingface
     embeddings:
-      - path: question
-        embedding: gte-small
-      - path:
-          - choices
-          - '*'
+      - path: lecture
         embedding: gte-small
     signals:
-      - path: question
+      - path: lecture
         signal:
           threshold: 0.85
           signal_name: near_dup
-      - path: question
+      - path: lecture
         signal:
           signal_name: pii
-      - path: question
-        signal:
-          split_by_paragraph: false
-          signal_name: lang_detection
-      - path: question
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: positive-sentiment
-          draft: main
-          signal_name: concept_score
-      - path: question
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: non-english
-          draft: main
-          signal_name: concept_score
-      - path: question
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: toxicity
-          draft: main
-          signal_name: concept_score
-      - path: question
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: question
-          draft: main
-          signal_name: concept_score
-      - path: question
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: legal-termination
-          draft: main
-          signal_name: concept_score
-      - path: question
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: source-code
-          draft: main
-          signal_name: concept_score
-      - path: question
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: negative-sentiment
-          draft: main
-          signal_name: concept_score
-      - path: question
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: profanity
-          draft: main
-          signal_name: concept_score
-      - path: question
-        signal:
-          signal_name: text_statistics
-      - path:
-          - choices
-          - '*'
-        signal:
-          threshold: 0.85
-          signal_name: near_dup
-      - path:
-          - choices
-          - '*'
-        signal:
-          signal_name: pii
-      - path:
-          - choices
-          - '*'
-        signal:
-          split_by_paragraph: false
-          signal_name: lang_detection
-      - path:
-          - choices
-          - '*'
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: positive-sentiment
-          draft: main
-          signal_name: concept_score
-      - path:
-          - choices
-          - '*'
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: non-english
-          draft: main
-          signal_name: concept_score
-      - path:
-          - choices
-          - '*'
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: toxicity
-          draft: main
-          signal_name: concept_score
-      - path:
-          - choices
-          - '*'
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: question
-          draft: main
-          signal_name: concept_score
-      - path:
-          - choices
-          - '*'
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: legal-termination
-          draft: main
-          signal_name: concept_score
-      - path:
-          - choices
-          - '*'
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: source-code
-          draft: main
-          signal_name: concept_score
-      - path:
-          - choices
-          - '*'
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: negative-sentiment
-          draft: main
-          signal_name: concept_score
-      - path:
-          - choices
-          - '*'
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: negative-sentiment
-          draft: main
-          signal_name: concept_score
-      - path:
-          - choices
-          - '*'
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: profanity
-          draft: main
-          signal_name: concept_score
-      - path:
-          - choices
-          - '*'
-        signal:
-          signal_name: text_statistics
-    settings:
-      ui:
-        media_paths:
-          - question
-          - - choices
-            - '*'
-        markdown_paths: []
-      preferred_embedding: gte-small
-  - namespace: lilac
-    name: pile-of-law-r-legaladvice
-    tags: []
-    source:
-      dataset_name: pile-of-law/pile-of-law
-      config_name: r_legaladvice
-      split: null
-      sample_size: null
-      revision: null
-      load_from_disk: false
-      source_name: huggingface
-    embeddings:
-      - path: text
-        embedding: gte-small
-    signals:
-      - path: text
-        signal:
-          threshold: 0.85
-          signal_name: near_dup
-      - path: text
-        signal:
-          signal_name: pii
-      - path: text
-        signal:
-          split_by_paragraph: false
-          signal_name: lang_detection
-      - path: text
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: positive-sentiment
-          draft: main
-          signal_name: concept_score
-      - path: text
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: non-english
-          draft: main
-          signal_name: concept_score
-      - path: text
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: toxicity
-          draft: main
-          signal_name: concept_score
-      - path: text
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: question
-          draft: main
-          signal_name: concept_score
-      - path: text
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: legal-termination
-          draft: main
-          signal_name: concept_score
-      - path: text
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: source-code
-          draft: main
-          signal_name: concept_score
-      - path: text
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: negative-sentiment
-          draft: main
-          signal_name: concept_score
-      - path: text
-        signal:
-          embedding: gte-small
-          namespace: lilac
-          concept_name: profanity
-          draft: main
-          signal_name: concept_score
-      - path: text
-        signal:
-          signal_name: text_statistics
-    settings:
-      ui:
-        media_paths:
-          - text
-        markdown_paths: []
-      preferred_embedding: gte-small
-  - namespace: lilac
-    name: science-qa-derek-thomas
-    tags: []
-    source:
-      dataset_name: derek-thomas/ScienceQA
-      config_name: null
-      split: null
-      sample_size: null
-      revision: null
-      load_from_disk: false
-      source_name: huggingface
-    embeddings:
-      - path: lecture
-        embedding: gte-small
-    signals:
-      - path: lecture
-        signal:
-          threshold: 0.85
-          signal_name: near_dup
-      - path: lecture
-        signal:
-          signal_name: pii
-      - path: lecture
+      - path: lecture
         signal:
           split_by_paragraph: false
           signal_name: lang_detection
@@ -2297,5 +1735,39 @@ datasets:
           - overview
         markdown_paths: []
       preferred_embedding: gte-small
+  - namespace: lilac
+    name: textbook_quality_programming
+    tags: []
+    source:
+      dataset_name: vikp/textbook_quality_programming
+      config_name: null
+      split: null
+      sample_size: null
+      revision: null
+      load_from_disk: false
+      source_name: huggingface
+    embeddings:
+      - path:
+          - outline
+          - '*'
+        embedding: gte-small
+      - path:
+          - concepts
+          - '*'
+        embedding: gte-small
+      - path: markdown
+        embedding: gte-small
+    signals: []
+    settings:
+      ui:
+        media_paths:
+          - - outline
+            - '*'
+          - - concepts
+            - '*'
+          - markdown
+        markdown_paths:
+          - markdown
+      preferred_embedding: gte-small
 signals: []
 concept_model_cache_embeddings: []

docker_start.sh

@@ -3,7 +3,7 @@
 # Fail if any of the commands below fail.
 set -e
 
-python docker_start.py
+lilac hf-docker-start
 gunicorn lilac.server:app \
   --bind 0.0.0.0:5432 \
   --preload -k uvicorn.workers.UvicornWorker \