Update app.py

app.py

@@ -1,67 +1,354 @@
 from fastapi import FastAPI, Request
 from fastapi.responses import HTMLResponse
-from transformers import pipeline
+import threading
+from email.header import decode_header
+import mysql.connector
+from transformers import pipeline, AutoTokenizer, AutoModelForCausalLM
+import email, imaplib, json, time
+import torch
 
-# Initialize the FastAPI app
 app = FastAPI()
 
-# Initialize the text generation pipeline
-pipe = pipeline("text2text-generation", model="google/flan-t5-small")
+# Global variable to control the main loop
+running = True
+IMAP_SERVER = 'imap.gmail.com'
+EMAIL_ADDRESS = '[email protected]'
+PASSWORD = 'gclc wsnx kywt uvqy '
+DB_CONFIG = {
+    'host': '0.tcp.in.ngrok.io',
+    'port': 18051,
+    'user': 'root',
+    'password': '',
+    'database': 'shipment_details'
+    }
 
 
+try:
+    # Check if GPU is available and set the device accordingly
+    # device = "cuda" if torch.cuda.is_available() else "cpu"
+    # print(f"Using device: {device}")
+    # Load the tokenizer
+    tokenizer = AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-128k-instruct", trust_remote_code=True, cache_dir='./phi3_mini')
+
+    # Load the model
+    model = AutoModelForCausalLM.from_pretrained(
+        "microsoft/Phi-3-mini-128k-instruct",
+        load_in_8bit=True,
+        device_map="auto",  # Automatically map layers to available device (GPU/CPU)
+        trust_remote_code=True,
+        cache_dir='./phi3_mini'
+    )
+
+except Exception as e:
+    print(f"Error initializing LLM: {e}")
+
+
+output_format = {
+    "origin": "",
+    "destination": "",
+    "Expected_shipment_datetime": "",
+    "Types of service": "",
+    "Warehouse": "",
+    "Description": "",
+    "Quantities": "",
+    "Carrier_details": ""
+  }
+
+prompt = f"""
+System prompt: You will be provided with an email containing shipment details. Your task is to extract specific information based on the given instructions.
+
+Instructions:
+1. The input email may contain irrelevant information. Focus only on extracting details about future shipments.
+2. The output should be in JSON format. If a type of information is not found, it should be marked as null.
+3. Extract the following information:
+    - origin: The origin location of the consignment.
+    - destination: The destination location of the consignment.
+    - expected_shipment_datetime: The expected date and time of delivery to the warehouse (format: yyyy-mm-dd hh:mm:ss).
+    - types_of_service: The type of service (AIR, LCL, FCL). AIR can be mentioned as flight, aeroplane, or any mode of air transport. LCL is a Less-Container Load, and FCL is a Full-Container Load.
+    - warehouse: The name of the warehouse.
+    - description: A brief description of the email (ASN) within 100 words and the responce is not truncated and covers the essential points.
+    - quantities: The number of items in the shipment.
+    - carrier_details: The details of the carrier.
+4. the output extracted information contains must be in this format:
+{{
+    "origin": "",
+    "destination": "",
+    "expected_shipment_datetime": "",
+    "types_of_service": "",
+    "warehouse": "",
+    "description": "",
+    "quantities": "",
+    "carrier_details": ""
+}}
+Examples:
+
+1. Email: We are pleased to inform you of an upcoming shipment originating from Hamburg and destined for New York. The shipment is expected to arrive on August 15, 2024. This consignment includes various electronics, with an estimated quantity of 200 units. The service type for this shipment is AIR, provided by our reliable carrier, Sky Logistics.
+   Extracted Information:
+        origin: Hamburg,
+        destination: New York,
+        expected_shipment_datetime: 2024-08-15 00:00:000,
+        types_of_service: AIR,
+        warehouse: Sky Logistics,
+        description: We are pleased to inform you of an upcoming shipment originating from Hamburg and destined for New York. The shipment is expected to arrive on August 15, 2024.,
+        quantities: 200 units,
+        carrier_details: Sky Logistics
+
+2. Email: Please be advised of a shipment from our supplier in Shanghai heading to Los Angeles. The expected date of arrival is July 30, 2024. The shipment consists of mixed goods, mainly textiles, with a total of 500 pieces. This delivery will be handled through LCL service by Ocean Freight Co.
+   Extracted Information:
+        origin: Shanghai,
+        destination: Los Angeles,
+        expected_shipment_datetime: 2024-07-30 00:00:0000,
+        types_of_service: LCL,
+        warehouse: Ocean Freight Co.,
+        description: Please be advised of a shipment from our supplier in Shanghai heading to Los Angeles. The expected date of arrival is July 30, 2024.,
+        quantities: 500 pieces,
+        carrier_details: Ocean Freight Co.
+
+3. Email: A new shipment is on its way from Mumbai to London, scheduled to reach by August 22, 2024. This batch contains furniture items, totaling 150 pieces. It is managed by Global Carriers.
+   Extracted Information:
+        origin: Mumbai,
+        destination: London,
+        expected_shipment_datetime: 2024-08-22 00:00:00000,
+        types_of_service: null,
+        warehouse: Global Carriers,
+        description: A new shipment is on its way from Mumbai to London, scheduled to reach by August 22, 2024.,
+        quantities: 150 pieces,
+        carrier_details: Global Carriers
+
+4. Email: We are notifying you about a shipment dispatched from Tokyo, heading towards Sydney, with an estimated arrival date of September 10, 2024. The cargo includes automotive parts, summing up to 350 units. This shipment will be transported via AIR service, operated by Jet Logistics.
+   Extracted Information:
+        origin: Tokyo,
+        destination: Sydney,
+        expected_shipment_datetime: 2024-09-10 00:00:0000,
+        types_of_service: AIR,
+        warehouse: Jet Logistics,
+        description: We are notifying you about a shipment dispatched from Tokyo, heading towards Sydney, with an estimated arrival date of September 10, 2024.,
+        quantities: 350 units,
+        carrier_details: Jet Logistics
+
+5. Email: Kindly note the details of a forthcoming shipment from Berlin to Toronto. The shipment encompasses various household goods, with a total quantity of 400 items. We have arranged for this to be shipped using LCL service, provided by Sea Wave Transport.
+   Extracted Information:
+        origin: Berlin,
+        destination: Toronto,
+        expected_shipment_datetime: null,
+        types_of_service: LCL,
+        warehouse: Sea Wave Transport,
+        description: Kindly note the details of a forthcoming shipment from Berlin to Toronto. The expected arrival is on August 5, 2024.,
+        quantities: 400 items,
+        carrier_details: Sea Wave Transport
+
+Output: {output_format}
+"""
+tput_format}
+
+def insert_data(extracted_details):
+    print(extracted_details)
+    try:
+        print('started !!')
+        # Initialize MySQL database connection
+        mydb = mysql.connector.connect(**DB_CONFIG)
+        cursor = mydb.cursor()
+        # print('db connecyed ! ''')
+        # Check if any of the required fields are empty
+        required_fields = [
+            'origin', 'destination', 'expected_shipment_datetime',
+            'types_of_service', 'warehouse', 'description',
+            'quantities', 'carrier_details'
+        ]
+        if all(extracted_details.get(field) in ["", None] for field in required_fields):
+            print("Skipping insertion: All specified extracted values are empty.")
+            return
+        sql = """
+        INSERT INTO shipment_details (origin, destination, expected_shipment_datetime, types_of_service, warehouse, description, quantities, carrier_details, sender, receiver, cc, bcc, subject)
+        VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
+        """
+        # print('ready to update ! ///')
+        val = (
+            extracted_details.get('origin'),
+            extracted_details.get('destination'),
+            extracted_details.get('expected_shipment_datetime'),
+            extracted_details.get('types_of_service'),
+            extracted_details.get('warehouse'),
+            extracted_details.get('description'),
+            extracted_details.get('quantities'),
+            extracted_details.get('carrier_details'),
+            extracted_details.get('sender'),
+            extracted_details.get('receiver'),
+            extracted_details.get('cc'),
+            extracted_details.get('bcc'),
+            extracted_details.get('subject')
+        )
+
+        cursor.execute(sql, val)
+        print('data inserted successfully ! ...')
+        mydb.commit()
+
+    except mysql.connector.Error as e:
+        print(f"Database error: {e}")
+    except Exception as e:
+        print(f"Error inserting data: {e}")
+        
+def get_details(mail):
+    pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)
+    prompt_ = f"\n[system]:{prompt}\n[User]: {mail}\n[Assistant]:"
+    output = pipe(prompt_, max_new_tokens=200)
+    # print(output[0])
+    # print(output[0]['generated_text'])
+
+    txt = output[0]['generated_text']
+    st = txt.find('[Assistant]:')
+    ed = txt.find('}')
+
+    data = txt[st+len('[Assistant]:'):].strip()
+
+    data = data[data.find('{'):data.find('}')+1].strip()
+
+    # print(data)
+    # jon = json.loads(data)
+    return data
+
+
+
+
+def read_email():
+    try:
+        mail = imaplib.IMAP4_SSL(IMAP_SERVER)
+        mail.login(EMAIL_ADDRESS, PASSWORD)
+        mail.select('inbox')
+
+        # Search for unread emails
+        status, messages = mail.search(None, 'UNSEEN')
+        message_ids = messages[0].split()
+        print(f"Total unread emails: {len(message_ids)}")
+
+        # Process each unread email
+        for message_id in message_ids:
+            try:
+                # Fetch the email
+                status, data = mail.fetch(message_id, '(RFC822)')
+                raw_email = data[0][1]
+                email_message = email.message_from_bytes(raw_email)
+
+                # Extract email metadata
+                sender = email_message['From']
+                receiver = email_message['To']
+                cc = email_message['Cc']
+                bcc = email_message['Bcc']
+                subject = email_message['Subject']
+                date = email_message['Date']
+
+                # print('sender : >>',type(sender), sender)
+                # Extract the email body
+                email_body = ""
+                if email_message.is_multipart():
+                    for part in email_message.walk():
+                        if part.get_content_type() == 'text/plain':
+                            email_body = part.get_payload(decode=True).decode('utf-8')
+                            break
+                else:
+                    email_body = email_message.get_payload(decode=True).decode('utf-8')
+                # print(email_body)
+                extracted_details_str = get_details(email_body)
+                # print('extracted mail info',type(extracted_details_str))
+                extracted_details = json.loads(extracted_details_str)
+                # print(extracted_details)
+                # print('loaded json data type',type(extracted_details))
+                # Combine metadata and extracted details
+                meta_data = {
+                    'sender': sender,
+                    'receiver': receiver,
+                    'cc': cc,
+                    'bcc': bcc,
+                    'subject': subject
+                }
+                print('meta_data',type(meta_data))
+                extracted_details.update(meta_data)
+
+                print('full data about email ! ...::',extracted_details)
+                insert_data(extracted_details)
+                print('email analysed succesfully !\n')
+            except Exception as e:
+                print(f"Error processing email ID {message_id}: {e}")
+
+        # Close the connection
+        mail.close()
+        mail.logout()
+
+    except Exception as e:
+        print(f"Error reading emails: {e}")
+        
+
+# Background loop for continuous email extraction
+def email_processing_loop():
+    global running
+    while running:
+        try:
+            read_email()
+            print("Emails processed")
+        except Exception as e:
+            print(f"Error in email processing: {e}")
+        time.sleep(60)  # Check emails every minute
+
+# HTML content embedded directly in app.py using Jinja2-style templating
+html_content = """
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Email Processing</title>
+    <style>
+        body { font-family: Arial, sans-serif; margin: 50px; }
+        h1 { color: #333; }
+        button {
+            padding: 10px 20px;
+            margin: 10px;
+            background-color: #4CAF50;
+            color: white;
+            border: none;
+            cursor: pointer;
+        }
+        button.stop { background-color: #f44336; }
+    </style>
+</head>
+<body>
+    <h1>Email Processing Status: {{ status }}</h1>
+
+    <!-- Display status -->
+    <button onclick="window.location.href='/start'">Start Processing</button>
+    <button class="stop" onclick="window.location.href='/stop'">Stop Processing</button>
+</body>
+</html>
+"""
+
+# Endpoint to display the home page with the email processor's status
 @app.get("/", response_class=HTMLResponse)
-def home():
-    """
-    Renders a simple HTML page with a text input field and a submit button.
-    """
-    return """
-    <!DOCTYPE html>
-    <html lang="en">
-    <head>
-        <meta charset="UTF-8">
-        <meta name="viewport" content="width=device-width, initial-scale=1.0">
-        <title>Text Generation</title>
-    </head>
-    <body>
-        <h1>Text Generation with FastAPI and Hugging Face Transformers</h1>
-        <form action="/generate" method="get">
-            <label for="text">Enter Text:</label><br><br>
-            <input type="text" id="text" name="text" required><br><br>
-            <button type="submit">Generate</button>
-        </form>
-        <div id="result"></div>
-    </body>
-    </html>
-    """
-
-
-@app.get("/generate", response_class=HTMLResponse)
-def generate(text: str):
-    """
-    Handles the text generation and returns the generated output in HTML format.
-    """
-    # Use the pipeline to generate text from the given input text
-    output = pipe(text)
-
-    # Prepare the result as an HTML response
-    generated_text = output[0]['generated_text']
-    return f"""
-    <!DOCTYPE html>
-    <html lang="en">
-    <head>
-        <meta charset="UTF-8">
-        <meta name="viewport" content="width=device-width, initial-scale=1.0">
-        <title>Generated Output</title>
-    </head>
-    <body>
-        <h1>Generated Text</h1>
-        <p><strong>Input:</strong> {text}</p>
-        <p><strong>Output:</strong> {generated_text}</p>
-        <br>
-        <a href="/">Go Back</a>
-    </body>
-    </html>
-    """
-
-
-# Run the app with: uvicorn your_filename:app --reload
+async def home():
+    global running
+    status = "Running" if running else "Stopped"
+    return HTMLResponse(content=html_content.replace("{{ status }}", status), status_code=200)
+
+# Endpoint to start the email processing loop
+@app.get("/start", response_class=HTMLResponse)
+async def start_processing():
+    global running
+    if not running:
+        running = True
+        threading.Thread(target=email_processing_loop, daemon=True).start()
+    status = "Running"
+    return HTMLResponse(content=html_content.replace("{{ status }}", status), status_code=200)
+
+# Endpoint to stop the email processing loop
+@app.get("/stop", response_class=HTMLResponse)
+async def stop_processing():
+    global running
+    running = False
+    status = "Stopped"
+    return HTMLResponse(content=html_content.replace("{{ status }}", status), status_code=200)
+
+# Start the email processing loop when the app runs
+if __name__ == "__main__":
+    # running = True
+    # threading.Thread(target=email_processing_loop, daemon=True).start()
+    import uvicorn
+    uvicorn.run()