Update app.py

app.py

@@ -13,7 +13,14 @@ from datetime import datetime
 import json
 import gradio as gr
 import re 
-from unsloth import FastLanguageModel
+# from unsloth import FastLanguageModel
+
+import transformers
+from transformers import BloomForCausalLM
+from transformers import BloomForTokenClassification
+from transformers import BloomForTokenClassification
+from transformers import BloomTokenizerFast
+import torch
 class DocumentRetrievalAndGeneration:
     def __init__(self, embedding_model_name, lm_model_id, data_folder):
         # hf_token = os.getenv('HF_TOKEN')
@@ -64,10 +71,10 @@ class DocumentRetrievalAndGeneration:
         return generate_text
     def initialize_llm2(self,model_id):
                 
-        tokenizer = AutoTokenizer.from_pretrained(model_id)
-        model = AutoModelForCausalLM.from_pretrained(model_id)
-        FastLanguageModel.for_inference(model) # Enable native 2x faster inference
-        
+        tokenizer = BloomTokenizerFast.from_pretrained("bigscience/bloom-1b3", local_files_only=True)
+        model = BloomForCausalLM.from_pretrained("bigscience/bloom-1b3", local_files_only=True)
+        result_length = 200
+        inputs = tokenizer(prompt, return_tensors="pt")
         # return generate_text
 
     def generate_response_with_timeout(self, model_inputs):
@@ -131,18 +138,9 @@ class DocumentRetrievalAndGeneration:
 
         # decoded = self.llm.tokenizer.batch_decode(generated_ids)
         # generated_response = decoded[0]
-
-        inputs = tokenizer(
-        [
-            alpaca_prompt.format(
-                "", # instruction
-                prompt, # input
-                "", # output - leave this blank for generation!
-            )
-        ], return_tensors = "pt")#.to("cuda")
+        generated_response=tokenizer.decode(model.generate(inputs["input_ids"], max_length=result_length,no_repeat_ngram_size=2)[0])
+        print(generated_response)
         
-        outputs = model.generate(**inputs, max_new_tokens = 64, use_cache = True)
-        tokenizer.batch_decode(outputs)
         match1 = re.search(r'\[/INST\](.*?)</s>', generated_response, re.DOTALL)
         
         match2 = re.search(r'Solution:(.*?)</s>', generated_response, re.DOTALL | re.IGNORECASE)