initial commit

.gitattributes

@@ -0,0 +1,4 @@
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+vocab.txt filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,2 @@
+.venv
+venv

README.md

@@ -0,0 +1,83 @@
+---
+pipeline_tag: sentence-similarity
+tags:
+- sentence-transformers
+- feature-extraction
+- sentence-similarity
+
+---
+
+# metarank/esci-MiniLM-L6-v2
+
+This is a [sentence-transformers](https://www.SBERT.net) model: It maps sentences & paragraphs to a 384 dimensional dense vector space and can be used for tasks like clustering or semantic search.
+
+A [sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) model fine-tuned on 
+[Amazon ESCI dataset](https://github.com/amazon-science/esci-data).
+
+## Usage (Sentence-Transformers)
+
+Using this model becomes easy when you have [sentence-transformers](https://www.SBERT.net) installed:
+
+```
+pip install -U sentence-transformers
+```
+
+Then you can use the model like this:
+
+```python
+from sentence_transformers import SentenceTransformer
+sentences = ["This is an example sentence", "Each sentence is converted"]
+
+model = SentenceTransformer('metarank/esci-MiniLM-L6-v2')
+embeddings = model.encode(sentences)
+print(embeddings)
+```
+
+## Training
+The model was trained with the parameters:
+
+**DataLoader**:
+
+`torch.utils.data.dataloader.DataLoader` of length 769 with parameters:
+```
+{'batch_size': 128, 'sampler': 'torch.utils.data.sampler.RandomSampler', 'batch_sampler': 'torch.utils.data.sampler.BatchSampler'}
+```
+
+**Loss**:
+
+`sentence_transformers.losses.MultipleNegativesRankingLoss.MultipleNegativesRankingLoss` with parameters:
+  ```
+  {'scale': 20.0, 'similarity_fct': 'cos_sim'}
+  ```
+
+Parameters of the fit()-Method:
+```
+{
+    "epochs": 1,
+    "evaluation_steps": 0,
+    "evaluator": "NoneType",
+    "max_grad_norm": 1,
+    "optimizer_class": "<class 'torch.optim.adamw.AdamW'>",
+    "optimizer_params": {
+        "lr": 2e-05
+    },
+    "scheduler": "WarmupLinear",
+    "steps_per_epoch": null,
+    "warmup_steps": 1000,
+    "weight_decay": 0.01
+}
+```
+
+
+## Full Model Architecture
+```
+SentenceTransformer(
+  (0): Transformer({'max_seq_length': 128, 'do_lower_case': False}) with Transformer model: BertModel 
+  (1): Pooling({'word_embedding_dimension': 384, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False})
+  (2): Normalize()
+)
+```
+
+## Citing & Authors
+
+* Roman Grebennikov

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "/home/shutty/.cache/torch/sentence_transformers/sentence-transformers_all-MiniLM-L6-v2/",
+  "architectures": [
+    "BertModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 384,
+  "initializer_range": 0.02,
+  "intermediate_size": 1536,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 6,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.27.4",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

config_sentence_transformers.json

@@ -0,0 +1,7 @@
+{
+  "__version__": {
+    "sentence_transformers": "2.0.0",
+    "transformers": "4.6.1",
+    "pytorch": "1.8.1"
+  }
+}

finetune.py

@@ -0,0 +1,82 @@
+from sentence_transformers import SentenceTransformer, LoggingHandler, util, models, evaluation, losses, InputExample, CrossEncoder
+from torch import nn
+import csv
+from torch.utils.data import DataLoader, Dataset
+import torch
+from sentence_transformers.evaluation import EmbeddingSimilarityEvaluator, SentenceEvaluator, SimilarityFunction, RerankingEvaluator
+from sentence_transformers.cross_encoder.evaluation import CERerankingEvaluator
+import logging
+import json
+import random
+import gzip
+
+model_name = 'sentence-transformers/all-MiniLM-L6-v2'
+
+train_batch_size = 128
+max_seq_length = 128
+num_epochs = 1
+warmup_steps = 1000
+model_save_path = '.'
+lr = 2e-5
+
+class ESCIDataset(Dataset):
+    def __init__(self, input):
+        self.queries = []
+        with gzip.open(input) as jsonfile:
+            for line in jsonfile.readlines():
+                query = json.loads(line)
+                for i in range(1,10):
+                    if len(query['e']) > 0 and len(query['i']) > 0:
+                        p = random.choice(query['e'])
+                        positive = p['title']
+                        n = random.choice(query['i'])
+                        negative = n['title']
+                        self.queries.append(InputExample(texts=[query['query'], positive, negative]))
+
+    def __getitem__(self, item):
+        return self.queries[item]
+
+    def __len__(self):
+        return len(self.queries)
+
+
+model = SentenceTransformer(model_name)
+model.max_seq_length = max_seq_length
+
+
+train_dataset = ESCIDataset(input='train-small.json.gz')
+eval_dataset = ESCIDataset(input='test-small.json.gz')
+train_dataloader = DataLoader(train_dataset, shuffle=True, batch_size=train_batch_size)
+train_loss = losses.MultipleNegativesRankingLoss(model=model)
+
+samples = {}
+for query in eval_dataset.queries:
+    qstr = query.texts[0]
+    sample = samples.get(qstr, {'query': qstr})
+    positive = sample.get('positive', [])
+    positive.append(query.texts[1])
+    sample['positive'] = positive
+    negative = sample.get('negative', [])
+    negative.append(query.texts[2])
+    sample['negative'] = negative
+    samples[qstr] = sample
+
+evaluator = RerankingEvaluator(samples=samples,name='esci')
+
+# Train the model
+
+model.fit(train_objectives=[(train_dataloader, train_loss)],
+          epochs=num_epochs,
+          warmup_steps=warmup_steps,
+          use_amp=True,
+          checkpoint_path=model_save_path,
+          checkpoint_save_steps=len(train_dataloader),
+          optimizer_params = {'lr': lr},
+#          evaluator=evaluator,
+#          evaluation_steps=300,
+          output_path=model_save_path
+          )
+
+# Save the model
+
+model.save(model_save_path)

modules.json

@@ -0,0 +1,20 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.models.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.models.Pooling"
+  },
+  {
+    "idx": 2,
+    "name": "2",
+    "path": "2_Normalize",
+    "type": "sentence_transformers.models.Normalize"
+  }
+]

onnx_convert.py

@@ -0,0 +1,18 @@
+from transformers import AutoTokenizer, AutoModel
+import torch
+
+max_seq_length=128
+
+model = AutoModel.from_pretrained(".")
+model.eval()
+
+inputs = {"input_ids": torch.ones(1, max_seq_length, dtype=torch.int64),
+        "attention_mask": torch.ones(1, max_seq_length, dtype=torch.int64),
+        "token_type_ids": torch.ones(1, max_seq_length, dtype=torch.int64)}
+
+symbolic_names = {0: 'batch_size', 1: 'max_seq_len'}
+
+torch.onnx.export(model, args=tuple(inputs.values()), f='pytorch_model.onnx', export_params=True, 
+    input_names=['input_ids', 'attention_mask', 'token_type_ids'], output_names=['last_hidden_state'],
+    dynamic_axes={'input_ids': symbolic_names, 'attention_mask': symbolic_names, 'token_type_ids': symbolic_names})
+

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8fd77ee0f10a3e91e0a3798f2b7b0ad1cf92538d062d089ea1fe83594dcdf5ab
+size 90891565

pytorch_model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9a477737434014d347f63a17bd90d092e3f0916e7b728e55ac887fd1b8b064c
+size 90984263

requirements.txt

@@ -0,0 +1,4 @@
+sentence-transformers==2.2.2
+torch==2.0.0
+onnx==1.13.1
+huggingface_hub==0.13.3

sentence_bert_config.json

@@ -0,0 +1,4 @@
+{
+  "max_seq_length": 128,
+  "do_lower_case": false
+}

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

test-small.json.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9542358cbd27f3d8ebcb41757384eb23e7c54a92ea2d8dbdc9e70d22ae5a0743
+size 69955958

tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "special_tokens_map_file": "/home/shutty/.cache/torch/sentence_transformers/sentence-transformers_all-MiniLM-L6-v2/special_tokens_map.json",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

train-small.json.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c7c4dd1391b2203b4d1c3d5618790ba3c01d9dcdaa1548b15a36ea4ee2d8840
+size 161159190

vocab.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07eced375cec144d27c900241f3e339478dec958f92fddbc551f295c992038a3
+size 231508