Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,202 @@
+---
+base_model: vidore/qwen2-warmup
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.11.1

adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "vidore/qwen2-warmup",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": "gaussian",
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": "(.*(model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$)",
+  "task_type": "FEATURE_EXTRACTION",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1ba254b682a2f6bca5678f2541387fa3ee92d1cf5034538c3d6c9d5dc6aa89d
+size 73911504

added_tokens.json

@@ -0,0 +1,16 @@
+{
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

chat_template.json

@@ -0,0 +1,3 @@
+{
+  "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"
+}

git_hash.txt

@@ -0,0 +1 @@
+cbe1183da8179ac2d1eaa856af4637157df81563

preprocessor_config.json

@@ -0,0 +1,29 @@
+{
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "Qwen2VLImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "max_pixels": 12845056,
+  "merge_size": 2,
+  "min_pixels": 3136,
+  "patch_size": 14,
+  "processor_class": "BiQwen2Processor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "max_pixels": 12845056,
+    "min_pixels": 3136
+  },
+  "temporal_patch_size": 2
+}

results_old.json

@@ -0,0 +1 @@
+{"validation_set": {"ndcg_at_1": 0.758, "ndcg_at_3": 0.82043, "ndcg_at_5": 0.83282, "ndcg_at_10": 0.84266, "ndcg_at_20": 0.85084, "ndcg_at_50": 0.8562, "ndcg_at_100": 0.85751, "map_at_1": 0.758, "map_at_3": 0.80533, "map_at_5": 0.81223, "map_at_10": 0.81638, "map_at_20": 0.81867, "map_at_50": 0.81961, "map_at_100": 0.81973, "recall_at_1": 0.758, "recall_at_3": 0.864, "recall_at_5": 0.894, "recall_at_10": 0.924, "recall_at_20": 0.956, "recall_at_50": 0.982, "recall_at_100": 0.99, "precision_at_1": 0.758, "precision_at_3": 0.288, "precision_at_5": 0.1788, "precision_at_10": 0.0924, "precision_at_20": 0.0478, "precision_at_50": 0.01964, "precision_at_100": 0.0099, "mrr_at_1": 0.758, "mrr_at_3": 0.8066666666666666, "mrr_at_5": 0.8126666666666666, "mrr_at_10": 0.8168103174603175, "mrr_at_20": 0.8190987982176062, "mrr_at_50": 0.8200907132188691, "mrr_at_100": 0.8201695678965017, "naucs_at_1_max": 0.8180302294498618, "naucs_at_1_std": -0.6400185657127382, "naucs_at_1_diff1": 0.9092042099472675, "naucs_at_3_max": 0.9025004266939748, "naucs_at_3_std": -0.7107157080275334, "naucs_at_3_diff1": 0.8724682824145202, "naucs_at_5_max": 0.8931988002058825, "naucs_at_5_std": -0.7867259464352299, "naucs_at_5_diff1": 0.8751930142164964, "naucs_at_10_max": 0.9073050272740686, "naucs_at_10_std": -0.9626025848936113, "naucs_at_10_diff1": 0.8942331318492321, "naucs_at_20_max": 0.9277438248026504, "naucs_at_20_std": -1.2098930481283448, "naucs_at_20_diff1": 0.8992233256939183, "naucs_at_50_max": 0.9854756717501884, "naucs_at_50_std": -1.1305114638448153, "naucs_at_50_diff1": 0.9709513435003653, "naucs_at_100_max": 1.0, "naucs_at_100_std": -1.1909430438842146, "naucs_at_100_diff1": 0.9738562091503188}, "syntheticDocQA_energy": {"ndcg_at_1": 0.75, "ndcg_at_3": 0.83833, "ndcg_at_5": 0.85424, "ndcg_at_10": 0.86014, "ndcg_at_20": 0.8652, "ndcg_at_50": 0.86954, "ndcg_at_100": 0.87114, "map_at_1": 0.75, "map_at_3": 0.82, "map_at_5": 0.8285, "map_at_10": 0.83061, "map_at_20": 0.832, "map_at_50": 0.83285, "map_at_100": 0.83299, "recall_at_1": 0.75, "recall_at_3": 0.89, "recall_at_5": 0.93, "recall_at_10": 0.95, "recall_at_20": 0.97, "recall_at_50": 0.99, "recall_at_100": 1.0, "precision_at_1": 0.75, "precision_at_3": 0.29667, "precision_at_5": 0.186, "precision_at_10": 0.095, "precision_at_20": 0.0485, "precision_at_50": 0.0198, "precision_at_100": 0.01, "mrr_at_1": 0.76, "mrr_at_3": 0.825, "mrr_at_5": 0.8334999999999999, "mrr_at_10": 0.8356111111111112, "mrr_at_20": 0.8370757575757576, "mrr_at_50": 0.8379469696969697, "mrr_at_100": 0.8380821048321049, "naucs_at_1_max": 0.5903087388801671, "naucs_at_1_std": -0.6087912087912085, "naucs_at_1_diff1": 0.8399162742019884, "naucs_at_3_max": 0.567605875783866, "naucs_at_3_std": -0.6042865733184407, "naucs_at_3_diff1": 0.8077914268533612, "naucs_at_5_max": 0.652260904361747, "naucs_at_5_std": -0.5182739762571643, "naucs_at_5_diff1": 0.7465652927837797, "naucs_at_10_max": 0.5948646125116731, "naucs_at_10_std": -0.89458450046684, "naucs_at_10_diff1": 0.7997198879551832, "naucs_at_20_max": 0.4609399315281665, "naucs_at_20_std": -0.7231559290382794, "naucs_at_20_diff1": 0.7587924058512269, "naucs_at_50_max": 1.0, "naucs_at_50_std": 0.7222222222222041, "naucs_at_50_diff1": 0.7222222222222041, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "syntheticDocQA_healthcare_industry": {"ndcg_at_1": 0.83, "ndcg_at_3": 0.90547, "ndcg_at_5": 0.92139, "ndcg_at_10": 0.92139, "ndcg_at_20": 0.92139, "ndcg_at_50": 0.92139, "ndcg_at_100": 0.92139, "map_at_1": 0.83, "map_at_3": 0.88667, "map_at_5": 0.89517, "map_at_10": 0.89517, "map_at_20": 0.89517, "map_at_50": 0.89517, "map_at_100": 0.89517, "recall_at_1": 0.83, "recall_at_3": 0.96, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.83, "precision_at_3": 0.32, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.83, "mrr_at_3": 0.8866666666666667, "mrr_at_5": 0.8956666666666666, "mrr_at_10": 0.8956666666666666, "mrr_at_20": 0.8956666666666666, "mrr_at_50": 0.8956666666666666, "mrr_at_100": 0.8956666666666666, "naucs_at_1_max": 0.4146978580135842, "naucs_at_1_std": -0.3873280315783354, "naucs_at_1_diff1": 0.8308179021303772, "naucs_at_3_max": 0.29470121381886233, "naucs_at_3_std": -1.2987861811391161, "naucs_at_3_diff1": 0.7741596638655437, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "syntheticDocQA_artificial_intelligence_test": {"ndcg_at_1": 0.79, "ndcg_at_3": 0.85417, "ndcg_at_5": 0.86621, "ndcg_at_10": 0.88356, "ndcg_at_20": 0.88356, "ndcg_at_50": 0.88764, "ndcg_at_100": 0.88764, "map_at_1": 0.79, "map_at_3": 0.83833, "map_at_5": 0.84483, "map_at_10": 0.85269, "map_at_20": 0.85269, "map_at_50": 0.8534, "map_at_100": 0.8534, "recall_at_1": 0.79, "recall_at_3": 0.9, "recall_at_5": 0.93, "recall_at_10": 0.98, "recall_at_20": 0.98, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.79, "precision_at_3": 0.3, "precision_at_5": 0.186, "precision_at_10": 0.098, "precision_at_20": 0.049, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.8, "mrr_at_3": 0.8433333333333333, "mrr_at_5": 0.8498333333333333, "mrr_at_10": 0.8576904761904761, "mrr_at_20": 0.8576904761904761, "mrr_at_50": 0.8583950216450217, "mrr_at_100": 0.8583950216450217, "naucs_at_1_max": 0.11376151201096464, "naucs_at_1_std": -0.618270132493328, "naucs_at_1_diff1": 0.868492557770457, "naucs_at_3_max": 0.40303454715219317, "naucs_at_3_std": -0.5295518207282941, "naucs_at_3_diff1": 0.8905228758169933, "naucs_at_5_max": 0.7246231826063763, "naucs_at_5_std": -0.17887154861944707, "naucs_at_5_diff1": 0.883286647992531, "naucs_at_10_max": 1.0, "naucs_at_10_std": 0.540149393090577, "naucs_at_10_diff1": 0.861111111111116, "naucs_at_20_max": 1.0, "naucs_at_20_std": 0.540149393090577, "naucs_at_20_diff1": 0.861111111111116, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "syntheticDocQA_government_reports": {"ndcg_at_1": 0.78, "ndcg_at_3": 0.88071, "ndcg_at_5": 0.89276, "ndcg_at_10": 0.89632, "ndcg_at_20": 0.89882, "ndcg_at_50": 0.89882, "ndcg_at_100": 0.89882, "map_at_1": 0.78, "map_at_3": 0.85667, "map_at_5": 0.86317, "map_at_10": 0.86483, "map_at_20": 0.8655, "map_at_50": 0.8655, "map_at_100": 0.8655, "recall_at_1": 0.78, "recall_at_3": 0.95, "recall_at_5": 0.98, "recall_at_10": 0.99, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.78, "precision_at_3": 0.31667, "precision_at_5": 0.196, "precision_at_10": 0.099, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.78, "mrr_at_3": 0.8583333333333333, "mrr_at_5": 0.8648333333333333, "mrr_at_10": 0.8665, "mrr_at_20": 0.8671666666666666, "mrr_at_50": 0.8671666666666666, "mrr_at_100": 0.8671666666666666, "naucs_at_1_max": 0.06732507998330828, "naucs_at_1_std": -0.2970278666481208, "naucs_at_1_diff1": 0.8174757731719745, "naucs_at_3_max": 0.24948646125116392, "naucs_at_3_std": -0.4772175536881391, "naucs_at_3_diff1": 0.7461251167133489, "naucs_at_5_max": 0.861111111111116, "naucs_at_5_std": -0.8576097105508901, "naucs_at_5_diff1": 0.8692810457516353, "naucs_at_10_max": 1.0, "naucs_at_10_std": -0.5634920634920767, "naucs_at_10_diff1": 0.8692810457516413, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "infovqa_subsampled": {"ndcg_at_1": 0.734, "ndcg_at_3": 0.80305, "ndcg_at_5": 0.81767, "ndcg_at_10": 0.83013, "ndcg_at_20": 0.83437, "ndcg_at_50": 0.83845, "ndcg_at_100": 0.84042, "map_at_1": 0.734, "map_at_3": 0.78733, "map_at_5": 0.79533, "map_at_10": 0.80057, "map_at_20": 0.80184, "map_at_50": 0.80254, "map_at_100": 0.80272, "recall_at_1": 0.734, "recall_at_3": 0.848, "recall_at_5": 0.884, "recall_at_10": 0.922, "recall_at_20": 0.938, "recall_at_50": 0.958, "recall_at_100": 0.97, "precision_at_1": 0.734, "precision_at_3": 0.28267, "precision_at_5": 0.1768, "precision_at_10": 0.0922, "precision_at_20": 0.0469, "precision_at_50": 0.01916, "precision_at_100": 0.0097, "mrr_at_1": 0.738, "mrr_at_3": 0.7893333333333333, "mrr_at_5": 0.7977333333333333, "mrr_at_10": 0.8026674603174603, "mrr_at_20": 0.8039375901875903, "mrr_at_50": 0.8046370416553112, "mrr_at_100": 0.8048174808869882, "naucs_at_1_max": 0.4981791117249281, "naucs_at_1_std": -0.17621422402298867, "naucs_at_1_diff1": 0.8717617044310272, "naucs_at_3_max": 0.6137597098616197, "naucs_at_3_std": 0.011626112454345153, "naucs_at_3_diff1": 0.8184191573640626, "naucs_at_5_max": 0.7200154103413223, "naucs_at_5_std": 0.21685137217613695, "naucs_at_5_diff1": 0.803772254828026, "naucs_at_10_max": 0.781033781033783, "naucs_at_10_std": 0.3903971845148331, "naucs_at_10_diff1": 0.7920227920227934, "naucs_at_20_max": 0.827444956477214, "naucs_at_20_std": 0.5863980000602383, "naucs_at_20_diff1": 0.7833047197373586, "naucs_at_50_max": 0.9540927482103958, "naucs_at_50_std": 0.848539415766309, "naucs_at_50_diff1": 0.754479569605618, "naucs_at_100_max": 0.9814814814814826, "naucs_at_100_std": 0.8747587924058554, "naucs_at_100_diff1": 0.8075319016495498}, "docvqa_subsampled": {"ndcg_at_1": 0.39, "ndcg_at_3": 0.462, "ndcg_at_5": 0.48489, "ndcg_at_10": 0.5097, "ndcg_at_20": 0.52735, "ndcg_at_50": 0.54211, "ndcg_at_100": 0.55028, "map_at_1": 0.39, "map_at_3": 0.44467, "map_at_5": 0.45727, "map_at_10": 0.46726, "map_at_20": 0.47209, "map_at_50": 0.47432, "map_at_100": 0.47506, "recall_at_1": 0.39, "recall_at_3": 0.512, "recall_at_5": 0.568, "recall_at_10": 0.646, "recall_at_20": 0.716, "recall_at_50": 0.792, "recall_at_100": 0.842, "precision_at_1": 0.39, "precision_at_3": 0.17067, "precision_at_5": 0.1136, "precision_at_10": 0.0646, "precision_at_20": 0.0358, "precision_at_50": 0.01584, "precision_at_100": 0.00842, "mrr_at_1": 0.39, "mrr_at_3": 0.4453333333333333, "mrr_at_5": 0.45793333333333336, "mrr_at_10": 0.467965873015873, "mrr_at_20": 0.472880522779594, "mrr_at_50": 0.47501925006532203, "mrr_at_100": 0.47575775396072806, "naucs_at_1_max": 0.6216522447712546, "naucs_at_1_std": -0.025235083362436097, "naucs_at_1_diff1": 0.7875493125347613, "naucs_at_3_max": 0.6606267941551014, "naucs_at_3_std": 0.03579137078845739, "naucs_at_3_diff1": 0.7017682545164144, "naucs_at_5_max": 0.74680386743496, "naucs_at_5_std": 0.09704386376914526, "naucs_at_5_diff1": 0.6693062444869572, "naucs_at_10_max": 0.7814818523315649, "naucs_at_10_std": 0.2227106676701153, "naucs_at_10_diff1": 0.6354380989602686, "naucs_at_20_max": 0.7874150557746986, "naucs_at_20_std": 0.265915964611915, "naucs_at_20_diff1": 0.6136687608890765, "naucs_at_50_max": 0.8287788049692807, "naucs_at_50_std": 0.4682394325251468, "naucs_at_50_diff1": 0.6185001065953447, "naucs_at_100_max": 0.8895765310556315, "naucs_at_100_std": 0.6355634458528353, "naucs_at_100_diff1": 0.6351289277334297}, "arxivqa_subsampled": {"ndcg_at_1": 0.726, "ndcg_at_3": 0.77528, "ndcg_at_5": 0.79362, "ndcg_at_10": 0.80637, "ndcg_at_20": 0.81796, "ndcg_at_50": 0.82404, "ndcg_at_100": 0.82704, "map_at_1": 0.726, "map_at_3": 0.76267, "map_at_5": 0.77297, "map_at_10": 0.7785, "map_at_20": 0.78165, "map_at_50": 0.78253, "map_at_100": 0.78281, "recall_at_1": 0.726, "recall_at_3": 0.812, "recall_at_5": 0.856, "recall_at_10": 0.894, "recall_at_20": 0.94, "recall_at_50": 0.972, "recall_at_100": 0.99, "precision_at_1": 0.726, "precision_at_3": 0.27067, "precision_at_5": 0.1712, "precision_at_10": 0.0894, "precision_at_20": 0.047, "precision_at_50": 0.01944, "precision_at_100": 0.0099, "mrr_at_1": 0.728, "mrr_at_3": 0.7646666666666666, "mrr_at_5": 0.7744666666666667, "mrr_at_10": 0.7800809523809523, "mrr_at_20": 0.7833235527888005, "mrr_at_50": 0.7841031955347688, "mrr_at_100": 0.7843909166550672, "naucs_at_1_max": 0.6997774230011768, "naucs_at_1_std": -0.5231569420328694, "naucs_at_1_diff1": 0.8633501528517435, "naucs_at_3_max": 0.6897337960513782, "naucs_at_3_std": -0.5291702816513941, "naucs_at_3_diff1": 0.8149586304674437, "naucs_at_5_max": 0.7054787119083646, "naucs_at_5_std": -0.5820793710827757, "naucs_at_5_diff1": 0.8136211368057061, "naucs_at_10_max": 0.7270912092007895, "naucs_at_10_std": -0.5280336510302989, "naucs_at_10_diff1": 0.8138810499973373, "naucs_at_20_max": 0.7532679738562081, "naucs_at_20_std": -0.6036103330221042, "naucs_at_20_diff1": 0.8214441332088407, "naucs_at_50_max": 0.7690742963852164, "naucs_at_50_std": -0.5972722422302272, "naucs_at_50_diff1": 0.8394357743097238, "naucs_at_100_max": 0.8846872082166138, "naucs_at_100_std": 0.28309990662931644, "naucs_at_100_diff1": 0.8627450980391941}, "tabfquad_subsampled": {"ndcg_at_1": 0.83929, "ndcg_at_3": 0.90098, "ndcg_at_5": 0.91297, "ndcg_at_10": 0.92095, "ndcg_at_20": 0.92194, "ndcg_at_50": 0.92194, "ndcg_at_100": 0.92255, "map_at_1": 0.83929, "map_at_3": 0.8875, "map_at_5": 0.89429, "map_at_10": 0.89752, "map_at_20": 0.89784, "map_at_50": 0.89784, "map_at_100": 0.8979, "recall_at_1": 0.83929, "recall_at_3": 0.93929, "recall_at_5": 0.96786, "recall_at_10": 0.99286, "recall_at_20": 0.99643, "recall_at_50": 0.99643, "recall_at_100": 1.0, "precision_at_1": 0.83929, "precision_at_3": 0.3131, "precision_at_5": 0.19357, "precision_at_10": 0.09929, "precision_at_20": 0.04982, "precision_at_50": 0.01993, "precision_at_100": 0.01, "mrr_at_1": 0.8428571428571429, "mrr_at_3": 0.8892857142857142, "mrr_at_5": 0.8960714285714286, "mrr_at_10": 0.8993055555555555, "mrr_at_20": 0.8996302308802309, "mrr_at_50": 0.8996302308802309, "mrr_at_100": 0.8996887788895985, "naucs_at_1_max": 0.6308196005675004, "naucs_at_1_std": 0.2706537160318682, "naucs_at_1_diff1": 0.8597730001091355, "naucs_at_3_max": 0.9026473334431836, "naucs_at_3_std": 0.6891580161476388, "naucs_at_3_diff1": 0.8670840885373794, "naucs_at_5_max": 0.935937337898122, "naucs_at_5_std": 0.7217553688141943, "naucs_at_5_diff1": 0.9255628177196795, "naucs_at_10_max": 1.0, "naucs_at_10_std": 0.6790382819794653, "naucs_at_10_diff1": 0.9346405228758269, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 0.8692810457516478, "naucs_at_50_max": 1.0, "naucs_at_50_std": 1.0, "naucs_at_50_diff1": 0.8692810457515607, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0}, "tatdqa": {"ndcg_at_1": 0.48346, "ndcg_at_3": 0.59726, "ndcg_at_5": 0.62772, "ndcg_at_10": 0.66144, "ndcg_at_20": 0.67603, "ndcg_at_50": 0.68527, "ndcg_at_100": 0.68807, "map_at_1": 0.48346, "map_at_3": 0.56835, "map_at_5": 0.58525, "map_at_10": 0.59945, "map_at_20": 0.60353, "map_at_50": 0.60508, "map_at_100": 0.60535, "recall_at_1": 0.48346, "recall_at_3": 0.6813, "recall_at_5": 0.75526, "recall_at_10": 0.85809, "recall_at_20": 0.91521, "recall_at_50": 0.96091, "recall_at_100": 0.97775, "precision_at_1": 0.48346, "precision_at_3": 0.2271, "precision_at_5": 0.15105, "precision_at_10": 0.08581, "precision_at_20": 0.04576, "precision_at_50": 0.01922, "precision_at_100": 0.00978, "mrr_at_1": 0.481659651232712, "mrr_at_3": 0.5672479454800562, "mrr_at_5": 0.5842653838444578, "mrr_at_10": 0.5985298609321459, "mrr_at_20": 0.6025970848960606, "mrr_at_50": 0.6041227644029902, "mrr_at_100": 0.6043871575907215, "naucs_at_1_max": 0.4061317386686219, "naucs_at_1_std": -0.10341123916343513, "naucs_at_1_diff1": 0.6374359397144861, "naucs_at_3_max": 0.4410840815843837, "naucs_at_3_std": -0.09865131952567925, "naucs_at_3_diff1": 0.5188284524540077, "naucs_at_5_max": 0.46563199590069476, "naucs_at_5_std": -0.04481989346322378, "naucs_at_5_diff1": 0.502964743301536, "naucs_at_10_max": 0.5971251813035713, "naucs_at_10_std": 0.17609454180900883, "naucs_at_10_diff1": 0.49762399924214884, "naucs_at_20_max": 0.7074578979929439, "naucs_at_20_std": 0.35131109907333347, "naucs_at_20_diff1": 0.48466509719700673, "naucs_at_50_max": 0.7898897400643674, "naucs_at_50_std": 0.5136437718598935, "naucs_at_50_diff1": 0.5049777692390409, "naucs_at_100_max": 0.9296593843449183, "naucs_at_100_std": 0.8481609120263548, "naucs_at_100_diff1": 0.6461841102700067}, "shift_project": {"ndcg_at_1": 0.55, "ndcg_at_3": 0.69488, "ndcg_at_5": 0.7151, "ndcg_at_10": 0.73829, "ndcg_at_20": 0.75364, "ndcg_at_50": 0.75588, "ndcg_at_100": 0.75919, "map_at_1": 0.55, "map_at_3": 0.66167, "map_at_5": 0.67267, "map_at_10": 0.68257, "map_at_20": 0.68688, "map_at_50": 0.68735, "map_at_100": 0.68766, "recall_at_1": 0.55, "recall_at_3": 0.79, "recall_at_5": 0.84, "recall_at_10": 0.91, "recall_at_20": 0.97, "recall_at_50": 0.98, "recall_at_100": 1.0, "precision_at_1": 0.55, "precision_at_3": 0.26333, "precision_at_5": 0.168, "precision_at_10": 0.091, "precision_at_20": 0.0485, "precision_at_50": 0.0196, "precision_at_100": 0.01, "mrr_at_1": 0.56, "mrr_at_3": 0.6666666666666667, "mrr_at_5": 0.6776666666666666, "mrr_at_10": 0.6888849206349207, "mrr_at_20": 0.6922836417967997, "mrr_at_50": 0.6927598322729902, "mrr_at_100": 0.6930715205846785, "naucs_at_1_max": 0.17372744506073132, "naucs_at_1_std": -0.1267991293968969, "naucs_at_1_diff1": 0.6048866109667903, "naucs_at_3_max": 0.480847380190925, "naucs_at_3_std": 0.14538197032726533, "naucs_at_3_diff1": 0.34732007598528425, "naucs_at_5_max": 0.6464776632302403, "naucs_at_5_std": 0.16319955817378487, "naucs_at_5_diff1": 0.2898870888561598, "naucs_at_10_max": 0.6763668430335094, "naucs_at_10_std": 0.13523187052598903, "naucs_at_10_diff1": 0.3826641767818238, "naucs_at_20_max": 0.7152194211017693, "naucs_at_20_std": 0.00513538748833041, "naucs_at_20_diff1": 0.5224089635854351, "naucs_at_50_max": 0.638188608776847, "naucs_at_50_std": 0.09337068160597661, "naucs_at_50_diff1": 0.3489729225023405, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}}

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:091aa7594dc2fcfbfa06b9e3c22a5f0562ac14f30375c13af7309407a0e67b8a
+size 11420371

tokenizer_config.json

@@ -0,0 +1,144 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "processor_class": "BiQwen2Processor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

training_config.yml

@@ -0,0 +1,65 @@
+config:
+  (): colpali_engine.trainer.colmodel_training.ColModelTrainingConfig
+  output_dir: !path ../../../models/biqwen2-warmup-256-newpad-1e
+  processor:
+    (): colpali_engine.utils.transformers_wrappers.AllPurposeWrapper
+    class_to_instanciate: !ext colpali_engine.models.BiQwen2Processor
+    pretrained_model_name_or_path:  "./models/colqwen2_base" # "./models/paligemma-3b-mix-448"
+    # max_length: 50
+
+  model:
+    (): colpali_engine.utils.transformers_wrappers.AllPurposeWrapper
+    class_to_instanciate: !ext colpali_engine.models.BiQwen2
+    pretrained_model_name_or_path: "./models/qwen2-warmup"
+    torch_dtype:  !ext torch.bfloat16
+    use_cache: false
+    attn_implementation: "flash_attention_2"
+#    device_map: "auto"
+#    quantization_config:
+#      (): transformers.BitsAndBytesConfig
+#      load_in_4bit: true
+#      bnb_4bit_quant_type: "nf4"
+#      bnb_4bit_compute_dtype:  "bfloat16"
+#      bnb_4bit_use_double_quant: true
+
+  dataset_loading_func: !ext colpali_engine.utils.dataset_transformation.load_train_set
+  eval_dataset_loader: !import ../data/test_data.yaml
+
+  # max_length: 50
+  run_eval: true
+  
+  loss_func:
+    (): colpali_engine.loss.bi_encoder_losses.BiPairwiseCELoss
+  tr_args:
+    (): transformers.training_args.TrainingArguments
+    output_dir: null
+    overwrite_output_dir: true
+    num_train_epochs: 1
+    per_device_train_batch_size: 64
+    gradient_checkpointing: true
+    gradient_checkpointing_kwargs: { "use_reentrant": false }
+    # 6 x 8 gpus = 48 batch size
+    # gradient_accumulation_steps: 4
+    per_device_eval_batch_size: 64
+    eval_strategy: "steps"
+    dataloader_num_workers: 8
+    # bf16: true
+    save_steps: 500
+    logging_steps: 10
+    eval_steps: 100
+    warmup_steps: 100
+    learning_rate: 5e-5
+    save_total_limit: 1
+    resume_from_checkpoint: false
+    # optim: "paged_adamw_8bit"
+  peft_config:
+    (): peft.LoraConfig
+    r: 32
+    lora_alpha: 32
+    lora_dropout: 0.1
+    init_lora_weights: "gaussian"
+    bias: "none"
+    task_type: "FEATURE_EXTRACTION"
+    target_modules: '(.*(model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$)'
+    # target_modules: '(.*(language_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)'
+