Update app.py

app.py

@@ -71,13 +71,13 @@ class DocumentRetrievalAndGeneration:
         return generate_text
     def initialize_llm2(self,model_id):
                 
-        model_name = "mistralai/Mistral-7B-Instruct-v0.2"
-        pipeline = transformers.pipeline(
-            "text-generation",
-            model=model_name,
-            model_kwargs={"torch_dtype": torch.bfloat16},
-            device="cpu",
-        )
+        # model_name = "mistralai/Mistral-7B-Instruct-v0.2"
+        # pipeline = transformers.pipeline(
+        #     "text-generation",
+        #     model=model_name,
+        #     model_kwargs={"torch_dtype": torch.bfloat16},
+        #     device="cpu",
+        # )
 
         
         # return generate_text
@@ -144,7 +144,11 @@ class DocumentRetrievalAndGeneration:
         Solution:"NO SOLUTION AVAILABLE"
         </s>
         """
-
+        messages = [
+            {"role": "user", "content": prompt},
+        ]
+        pipe = pipeline("text-generation", model="mistralai/Mistral-7B-Instruct-v0.2")
+        generated_response=pipe(messages)
         # messages = [{"role": "user", "content": prompt}]
         # encodeds = self.llm.tokenizer.apply_chat_template(messages, return_tensors="pt")
         # model_inputs = encodeds.to(self.llm.device)
@@ -155,44 +159,45 @@ class DocumentRetrievalAndGeneration:
 
         # decoded = self.llm.tokenizer.batch_decode(generated_ids)
         # generated_response = decoded[0]
-        messages = []
-        # Check if history is None or empty and handle accordingly
-        if history:
-            for user_msg, assistant_msg in history:
-                messages.append({"role": "user", "content": user_msg})
-                messages.append({"role": "assistant", "content": assistant_msg})
+        #########################################################
+        # messages = []
+        # # Check if history is None or empty and handle accordingly
+        # if history:
+        #     for user_msg, assistant_msg in history:
+        #         messages.append({"role": "user", "content": user_msg})
+        #         messages.append({"role": "assistant", "content": assistant_msg})
         
-        # Always add the current user message
-        messages.append({"role": "user", "content": message})
+        # # Always add the current user message
+        # messages.append({"role": "user", "content": message})
         
-        # Construct the prompt using the pipeline's tokenizer
-        prompt = pipeline.tokenizer.apply_chat_template(
-            messages,
-            tokenize=False,
-            add_generation_prompt=True
-        )
+        # # Construct the prompt using the pipeline's tokenizer
+        # prompt = pipeline.tokenizer.apply_chat_template(
+        #     messages,
+        #     tokenize=False,
+        #     add_generation_prompt=True
+        # )
     
-        # Generate the response
-        terminators = [
-            pipeline.tokenizer.eos_token_id,
-            pipeline.tokenizer.convert_tokens_to_ids("")
-        ]
+        # # Generate the response
+        # terminators = [
+        #     pipeline.tokenizer.eos_token_id,
+        #     pipeline.tokenizer.convert_tokens_to_ids("")
+        # ]
     
-        # Adjust the temperature slightly above given to ensure variety
-        adjusted_temp = temperature + 0.1
+        # # Adjust the temperature slightly above given to ensure variety
+        # adjusted_temp = temperature + 0.1
     
-        # Generate outputs with adjusted parameters
-        outputs = pipeline(
-            prompt,
-            max_new_tokens=max_new_tokens,
-            do_sample=True,
-            temperature=adjusted_temp,
-            top_p=0.9
-        )
+        # # Generate outputs with adjusted parameters
+        # outputs = pipeline(
+        #     prompt,
+        #     max_new_tokens=max_new_tokens,
+        #     do_sample=True,
+        #     temperature=adjusted_temp,
+        #     top_p=0.9
+        # )
     
-        # Extract the generated text, skipping the length of the prompt
-        generated_text = outputs[0]["generated_text"]
-        generated_response = generated_text[len(prompt):]
+        # # Extract the generated text, skipping the length of the prompt
+        # generated_text = outputs[0]["generated_text"]
+        # generated_response = generated_text[len(prompt):]
         
         match1 = re.search(r'\[/INST\](.*?)</s>', generated_response, re.DOTALL)