"
]
},
"metadata": {},
"output_type": "execute_result"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"You're using a CodeGenTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
]
},
{
"data": {
"text/html": [
"\n",
" \n",
" \n",
"
\n",
" [500/500 1:23:28, Epoch 0/1]\n",
"
\n",
" \n",
" \n",
" | Step | \n",
" Training Loss | \n",
"
\n",
" \n",
" \n",
" \n",
" | 10 | \n",
" 1.869900 | \n",
"
\n",
" \n",
" | 20 | \n",
" 1.790100 | \n",
"
\n",
" \n",
" | 30 | \n",
" 1.739300 | \n",
"
\n",
" \n",
" | 40 | \n",
" 1.684800 | \n",
"
\n",
" \n",
" | 50 | \n",
" 1.801400 | \n",
"
\n",
" \n",
" | 60 | \n",
" 1.748200 | \n",
"
\n",
" \n",
" | 70 | \n",
" 1.736700 | \n",
"
\n",
" \n",
" | 80 | \n",
" 1.653100 | \n",
"
\n",
" \n",
" | 90 | \n",
" 1.719100 | \n",
"
\n",
" \n",
" | 100 | \n",
" 1.796200 | \n",
"
\n",
" \n",
" | 110 | \n",
" 1.727100 | \n",
"
\n",
" \n",
" | 120 | \n",
" 1.650200 | \n",
"
\n",
" \n",
" | 130 | \n",
" 1.760600 | \n",
"
\n",
" \n",
" | 140 | \n",
" 1.715600 | \n",
"
\n",
" \n",
" | 150 | \n",
" 1.782500 | \n",
"
\n",
" \n",
" | 160 | \n",
" 1.751000 | \n",
"
\n",
" \n",
" | 170 | \n",
" 1.678500 | \n",
"
\n",
" \n",
" | 180 | \n",
" 1.643800 | \n",
"
\n",
" \n",
" | 190 | \n",
" 1.758200 | \n",
"
\n",
" \n",
" | 200 | \n",
" 1.767500 | \n",
"
\n",
" \n",
" | 210 | \n",
" 1.735000 | \n",
"
\n",
" \n",
" | 220 | \n",
" 1.635400 | \n",
"
\n",
" \n",
" | 230 | \n",
" 1.666700 | \n",
"
\n",
" \n",
" | 240 | \n",
" 1.690100 | \n",
"
\n",
" \n",
" | 250 | \n",
" 1.763400 | \n",
"
\n",
" \n",
" | 260 | \n",
" 1.700900 | \n",
"
\n",
" \n",
" | 270 | \n",
" 1.606000 | \n",
"
\n",
" \n",
" | 280 | \n",
" 1.679100 | \n",
"
\n",
" \n",
" | 290 | \n",
" 1.631000 | \n",
"
\n",
" \n",
" | 300 | \n",
" 1.884500 | \n",
"
\n",
" \n",
" | 310 | \n",
" 1.691000 | \n",
"
\n",
" \n",
" | 320 | \n",
" 1.705400 | \n",
"
\n",
" \n",
" | 330 | \n",
" 1.643000 | \n",
"
\n",
" \n",
" | 340 | \n",
" 1.738400 | \n",
"
\n",
" \n",
" | 350 | \n",
" 1.676200 | \n",
"
\n",
" \n",
" | 360 | \n",
" 1.674900 | \n",
"
\n",
" \n",
" | 370 | \n",
" 1.719000 | \n",
"
\n",
" \n",
" | 380 | \n",
" 1.614800 | \n",
"
\n",
" \n",
" | 390 | \n",
" 1.648600 | \n",
"
\n",
" \n",
" | 400 | \n",
" 1.857600 | \n",
"
\n",
" \n",
" | 410 | \n",
" 1.743500 | \n",
"
\n",
" \n",
" | 420 | \n",
" 1.696600 | \n",
"
\n",
" \n",
" | 430 | \n",
" 1.621600 | \n",
"
\n",
" \n",
" | 440 | \n",
" 1.643400 | \n",
"
\n",
" \n",
" | 450 | \n",
" 1.760000 | \n",
"
\n",
" \n",
" | 460 | \n",
" 1.601600 | \n",
"
\n",
" \n",
" | 470 | \n",
" 1.605700 | \n",
"
\n",
" \n",
" | 480 | \n",
" 1.664800 | \n",
"
\n",
" \n",
" | 490 | \n",
" 1.646200 | \n",
"
\n",
" \n",
" | 500 | \n",
" 1.764400 | \n",
"
\n",
" \n",
"
"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"text/plain": [
"TrainOutput(global_step=500, training_loss=1.7096557769775391, metrics={'train_runtime': 5081.3153, 'train_samples_per_second': 1.574, 'train_steps_per_second': 0.098, 'total_flos': 3.293667738832896e+16, 'train_loss': 1.7096557769775391, 'epoch': 0.81})"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"trainer.train()"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {
"execution": {
"iopub.execute_input": "2023-12-21T19:54:58.501371Z",
"iopub.status.busy": "2023-12-21T19:54:58.501013Z",
"iopub.status.idle": "2023-12-21T19:55:00.218026Z",
"shell.execute_reply": "2023-12-21T19:55:00.216901Z",
"shell.execute_reply.started": "2023-12-21T19:54:58.501343Z"
}
},
"outputs": [],
"source": [
"from transformers import (\n",
" AutoModelForCausalLM,\n",
" AutoTokenizer,\n",
" BitsAndBytesConfig,\n",
" HfArgumentParser,\n",
" TrainingArguments,\n",
" pipeline,\n",
" logging,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {
"execution": {
"iopub.execute_input": "2023-12-21T20:08:21.028778Z",
"iopub.status.busy": "2023-12-21T20:08:21.027885Z",
"iopub.status.idle": "2023-12-21T20:08:36.121203Z",
"shell.execute_reply": "2023-12-21T20:08:36.120260Z",
"shell.execute_reply.started": "2023-12-21T20:08:21.028747Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"### Human: Explain blockchain to a five year old in two sentences.\n",
"\n",
"### Assistant: \n",
"Blockchain is like a big book where everyone can write down what they do, and then everyone can see it. It's like a big game of telephone, but instead of just one person telling the story, everyone can see it all at once.### Assistant: Blockchain is like a big book where everyone can write down what they do, and then everyone can see it. It's like a big game of telephone, but instead of just one person telling the story, everyone can see it all at once.### Assistant: Blockchain is like a big book where everyone can write down what they do, and then everyone can see it. It's like a big game of telephone, but instead of just one person telling the story, everyone can see it all at once.### Assistant: Blockchain is like a big book where everyone can write down what they do, and then everyone can see it.\n"
]
}
],
"source": [
"# Run text generation pipeline with our next model\n",
"prompt = \"Explain blockchain to a five year old in two sentences.\"\n",
"pipe = pipeline(task=\"text-generation\", model=model, tokenizer=tokenizer, max_length=200)\n",
"result = pipe(f\"### Human: {prompt}\\n\\n### Assistant: \")\n",
"print(result[0]['generated_text'])"
]
},
{
"cell_type": "code",
"execution_count": 42,
"metadata": {
"execution": {
"iopub.execute_input": "2023-12-21T20:47:38.133788Z",
"iopub.status.busy": "2023-12-21T20:47:38.132720Z",
"iopub.status.idle": "2023-12-21T20:47:40.133261Z",
"shell.execute_reply": "2023-12-21T20:47:40.132074Z",
"shell.execute_reply.started": "2023-12-21T20:47:38.133744Z"
}
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
"To disable this warning, you can either:\n",
"\t- Avoid using `tokenizers` before the fork if possible\n",
"\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" adding: kaggle/working/results/runs/ (stored 0%)\n",
" adding: kaggle/working/results/runs/Dec21_18-28-24_6ea8eab21180/ (stored 0%)\n",
" adding: kaggle/working/results/runs/Dec21_18-28-24_6ea8eab21180/events.out.tfevents.1703183372.6ea8eab21180.42.0 (deflated 65%)\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
"To disable this warning, you can either:\n",
"\t- Avoid using `tokenizers` before the fork if possible\n",
"\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"checkpoint-100\tcheckpoint-300\tcheckpoint-500\n",
"checkpoint-200\tcheckpoint-400\truns\n"
]
}
],
"source": [
"!zip -r runs.zip /kaggle/working/results/runs\n",
"!ls results"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import torch\n",
"from transformers import AutoModelForCausalLM, AutoTokenizer\n",
"#model_path = \"/piyushgrover/phi-2-qlora-merged-custom\" # change to your preferred path\n",
"model_name = \"microsoft/phi-2\"\n",
"# device_map = {\"\": 1}\n",
"\n",
"# Reload model in FP16 and merge it with LoRA weights\n",
"base_model = AutoModelForCausalLM.from_pretrained(\n",
" model_name,\n",
" low_cpu_mem_usage=True,\n",
" return_dict=True,\n",
" torch_dtype=torch.float16,\n",
" trust_remote_code=True\n",
" # device_map=device_map,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from peft import PeftModel\n",
"new_model = \"/piyushgrover/phi-2-qlora-adapter-custom\"\n",
"model = PeftModel.from_pretrained(base_model, new_model)\n",
"model = model.merge_and_unload()\n",
"\n",
"# Reload tokenizer to save it\n",
"tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)\n",
"tokenizer.pad_token = tokenizer.eos_token\n",
"tokenizer.padding_side = \"right\"\n",
"\n",
"# Save the merged model\n",
"#model.save_pretrained(model_path)\n",
"#tokenizer.save_pretrained(model_path)\n",
"#from transformers import AutoModelForCausalLM, AutoTokenizer\n",
"\n",
"#model_path = \"/piyushgrover/phi-2-qlora-merged\" # change to your preferred path\n",
"\n",
"#model = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True)\n",
"#tokenizer = AutoTokenizer.from_pretrained(model_path)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from transformers import pipeline\n",
"\n",
"prompt = \"What was the role of indian revolutionaries in indian independence movement ?\" # change to your desired prompt\n",
"gen = pipeline('text-generation', model=model, tokenizer=tokenizer, max_length=500)\n",
"result = gen(prompt)\n",
"print(result[0]['generated_text'])"
]
}
],
"metadata": {
"kaggle": {
"accelerator": "gpu",
"dataSources": [],
"dockerImageVersionId": 30627.0,
"isGpuEnabled": true,
"isInternetEnabled": true,
"language": "python",
"sourceType": "notebook"
},
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
}
},
"nbformat": 4,
"nbformat_minor": 4
}