Upload folder using huggingface_hub

README.md

@@ -1,3 +1,57 @@
 ---
+library_name: transformers
+base_model: meta-llama/Llama-2-7b-hf
 license: llama2
+pipeline_tag: text-generation
+language:
+- multilingual
+datasets:
+- cis-lmu/Glot500
 ---
+
+# MaLA-500: Massive Language Adaptation of Large Language Models
+
+MaLA-500 is a novel large language model designed to cover an extensive range of 534 languages. This model builds upon LLaMA 2 7B and integrates continued pretraining with vocabulary extension, with an expanded vocabulary size of 260,164, and LoRA low-rank adaptation.
+
+
+- **Continued Pretraining:** Enhances the model's ability to adapt to a wide range of languages.
+- **LoRA Low-Rank Adaptation:** LoRA low-rank adaptation refines the model's adaptation capabilities.
+- **Vocabulary Extension:** MaLA-500 boasts an extended vocabulary size of 260,164.
+- **Multilingual Proficiency:** Trained on Glot500-c, covering 534 languages.
+
+With vocabulary extension and LoRA modules, the MaLA-500 introduces additional 2.1B trainable parameters, making the total parameters to be 10.7B.
+
+Please refer to [our paper](https://arxiv.org/pdf/2401.13303.pdf) for more details.
+
+## How to Get Started with the Model
+
+Requirements:
+```
+transformers>=4.36.1
+peft>=0.6.2
+```
+
+Use the code below to get started with the model.
+
+``` python
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from peft import PeftModel
+
+base_model = AutoModelForCausalLM.from_pretrained('meta-llama/Llama-2-7b-hf')
+base_model.resize_token_embeddings(260164)
+tokenizer = AutoTokenizer.from_pretrained('MaLA-LM/mala-500-10b')
+model = PeftModel.from_pretrained(base_model, 'MaLA-LM/mala-500-10b')
+```
+
+## Citation
+
+```
+@misc{lin2024mala500,
+      title={MaLA-500: Massive Language Adaptation of Large Language Models}, 
+      author={Peiqin Lin and Shaoxiong Ji and Jörg Tiedemann and André F. T. Martins and Hinrich Schütze},
+      year={2024},
+      eprint={2401.13303},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+```

adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-2-7b-hf",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 32.0,
+  "lora_dropout": 0.1,
+  "modules_to_save": [
+    "embed_tokens",
+    "lm_head"
+  ],
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3607bf5fe0e4f5da8ce537530404eadf88d969c7f56ccdd0bcf9f5af7a4ca44d
+size 4270933608

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:728e0df348a26f222dc309d5fc27408d659aaab88e1d822d6cba2c4500324e76
+size 4455279

tokenizer_config.json

@@ -0,0 +1,42 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false,
+  "use_fast": true
+}