initial commit

README.md

@@ -1,12 +1,25 @@
 ---
-title: Retrieve
-emoji: 📚
-colorFrom: yellow
-colorTo: red
+title: PyTerrier Retrieve
+emoji: 🐕
+colorFrom: green
+colorTo: green
 sdk: gradio
-sdk_version: 3.8.2
+sdk_version: 3.7
 app_file: app.py
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# 🐕 PyTerrier: Retrieve
+
+This is a demonstration of [PyTerrier's TerrierRetrieve transformer](https://pyterrier.readthedocs.io/en/latest/terrier-retrieval.html).
+
+TerrierRetrieve functions as a `Q→R` (retrieval, query-to-result) transformer and can be used in pipelines accordingly. For example, you can
+pipe the results to a transformer such as `get_text` to load the text associated with the document:
+
+<div class="pipeline">
+  <div class="df" title="Query Frame">Q</div>
+  <div class="transformer attn" title="PisaRetrieve Transformer">TerrierRetrieve</div>
+  <div class="df" title="Result Frame">R</div>
+  <div class="transformer" title="get_text Transformer">get_text</div>
+  <div class="df" title="Result Frame">R</div>
+</div>

app.py

@@ -0,0 +1,64 @@
+import pandas as pd
+import gradio as gr
+import pyterrier as pt
+pt.init()
+from pyterrier_gradio import Demo, MarkdownFile, interface, df2code, code2md, EX_Q
+
+retr = pt.TerrierRetrieve.from_dataset('msmarco_passage', 'terrier_stemmed')
+
+COLAB_NAME = 'pyterrier_retrieve.ipynb'
+COLAB_INSTALL = '''
+!pip install -q python-terrier
+'''.strip()
+
+def predict(input, _, wmodel, num_results, pipe_text):
+  retr.controls["wmodel"] = wmodel
+  retr.controls["end"] = str(num_results -1)
+  code = f'''import pandas as pd
+import pyterrier as pt ; pt.init()
+
+retr = pt.TerrierRetrieve.from_dataset('msmarco_passage', 'terrier_stemmed', wmodel={repr(wmodel)}, num_results={num_results})
+'''
+  pipeline = retr
+  if pipe_text:
+    pipeline = pipeline >> pt.text.get_text(pt.get_dataset('irds:msmarco-passage'), 'text')
+    code += f'''
+pipeline = retr >> pt.text.get_text(pt.get_dataset('irds:msmarco-passage'), 'text')
+
+pipeline({df2code(input)})'''
+  else:
+    code += f'''
+retr({df2code(input)})'''
+  res = pipeline(input)
+  res['score'] = res['score'].map(lambda x: round(x, 2))
+  return (res, code2md(code, COLAB_INSTALL, COLAB_NAME))
+
+interface(
+  MarkdownFile('README.md'),
+  Demo(
+    predict,
+    {k: v for k, v in EX_Q.items() if k != 'antique/train'},
+    [
+    gr.Dropdown(
+      choices=['msmarco-passage stemmed'],
+      value='msmarco-passage stemmed',
+      label='Index',
+      interactive=False,
+    ), gr.Dropdown(
+      choices=['TF_IDF', 'BM25', 'PL2', 'DPH'],
+      value='BM25',
+      label='Retrieval Model',
+    ), gr.Slider(
+      minimum=1,
+      maximum=10,
+      value=5,
+      step=1.,
+      label='# Results'
+    ), gr.Checkbox(
+      value=True,
+      label="Include get_text in pipeline",
+    )],
+    scale=2/3
+  ),
+  MarkdownFile('wrapup.md'),
+).launch(share=True)

packages.txt

@@ -0,0 +1,5 @@
+openjdk-11-jdk
+openjdk-11-jre-headless
+openjdk-11-jre
+openjdk-11-jre-headless
+debianutils

requirements.txt

@@ -0,0 +1,5 @@
+git+https://github.com/seanmacavaney/[email protected]
+git+https://github.com/terrier-org/pyterrier
+pyterrier-pisa
+ir_datasets
+ir_measures

wrapup.md

@@ -0,0 +1,3 @@
+### References & Credits
+
+ - Craig Macdonald, Nicola Tonellotto, Sean MacAvaney, Iadh Ounis. [PyTerrier: Declarative Experimentation in Python from BM25 to Dense Retrieval](https://dl.acm.org/doi/abs/10.1145/3459637.3482013). CIKM 2021.