app.py

from fastapi import FastAPI, Request
from fastapi.responses import HTMLResponse
import threading
from email.header import decode_header
import mysql.connector
from transformers import pipeline, AutoTokenizer, AutoModelForCausalLM
import email, imaplib, json, time
import torch, logging
import uvicorn
from pydantic import BaseModel 
import pandas as pd
from llama_cpp import Llama

app = FastAPI()

# Configure logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

# Email and database configuration
DB_CONFIG = {
    'host': '0.tcp.in.ngrok.io',
    'port': 19297,
    'user': 'root',
    'password': '',
    'database': 'shipment_details'
    }

output_format = {
    "origin": "",
    "destination": "",
    "Expected_shipment_datetime": "",
    "Types of service": "",
    "Warehouse": "",
    "Description": "",
    "Quantities": "",
    "Carrier_details": ""
  }

prompt = f"""
System prompt: You will be provided with an email containing shipment details. Your task is to extract specific information based on the given instructions.

Instructions:
1. The input email may contain irrelevant information. Focus only on extracting details about future shipments.
2. The output should be in JSON format with all values to use only double qoutes only. If a type of information is not found, it should be marked as null.
3. Extract the following information:
    - origin: The origin location of the consignment.
    - destination: The destination location of the consignment.
    - expected_shipment_datetime: The expected date and time of delivery to the warehouse (format: yyyy-mm-dd hh:mm:ss).
    - types_of_service: The type of service (AIR, LCL, FCL). AIR can be mentioned as flight, aeroplane, or any mode of air transport. LCL is a Less-Container Load, and FCL is a Full-Container Load.
    - warehouse: The name of the warehouse.
    - description: A brief description of the email (ASN).
    - quantities: The number of items in the shipment.
    - carrier_details: The details of the carrier.
4. the output extracted information contains must be in this format:
{{
    "origin": "",
    "destination": "",
    "expected_shipment_datetime": "",
    "types_of_service": "",
    "warehouse": "",
    "description": "",
    "quantities": "",
    "carrier_details": ""
}}

Examples:

1. Email: We are pleased to inform you of an upcoming shipment originating from Hamburg and destined for New York. The shipment is expected to arrive on August 15, 2024. This consignment includes various electronics, with an estimated quantity of 200 units. The service type for this shipment is AIR, provided by our reliable carrier, Sky Logistics.
   Extracted Information:
        origin: Hamburg,
        destination: New York,
        expected_shipment_datetime: 2024-08-15 00:00:000,
        types_of_service: AIR,
        warehouse: Sky Logistics,
        description: We are pleased to inform you of an upcoming shipment originating from Hamburg and destined for New York. The shipment is expected to arrive on August 15, 2024.,
        quantities: 200 units,
        carrier_details: Sky Logistics

2. Email: Please be advised of a shipment from our supplier in Shanghai heading to Los Angeles. The expected date of arrival is July 30, 2024. The shipment consists of mixed goods, mainly textiles, with a total of 500 pieces. This delivery will be handled through LCL service by Ocean Freight Co.
   Extracted Information:
        origin: Shanghai,
        destination: Los Angeles,
        expected_shipment_datetime: 2024-07-30 00:00:0000,
        types_of_service: LCL,
        warehouse: Ocean Freight Co.,
        description: Please be advised of a shipment from our supplier in Shanghai heading to Los Angeles. The expected date of arrival is July 30, 2024.,
        quantities: 500 pieces,
        carrier_details: Ocean Freight Co.

3. Email: A new shipment is on its way from Mumbai to London, scheduled to reach by August 22, 2024. This batch contains furniture items, totaling 150 pieces. It is managed by Global Carriers.
   Extracted Information:
        origin: Mumbai,
        destination: London,
        expected_shipment_datetime: 2024-08-22 00:00:00000,
        types_of_service: null,
        warehouse: Global Carriers,
        description: A new shipment is on its way from Mumbai to London, scheduled to reach by August 22, 2024.,
        quantities: 150 pieces,
        carrier_details: Global Carriers

4. Email: We are notifying you about a shipment dispatched from Tokyo, heading towards Sydney, with an estimated arrival date of September 10, 2024. The cargo includes automotive parts, summing up to 350 units. This shipment will be transported via AIR service, operated by Jet Logistics.
   Extracted Information:
        origin: Tokyo,
        destination: Sydney,
        expected_shipment_datetime: 2024-09-10 00:00:0000,
        types_of_service: AIR,
        warehouse: Jet Logistics,
        description: We are notifying you about a shipment dispatched from Tokyo, heading towards Sydney, with an estimated arrival date of September 10, 2024.,
        quantities: 350 units,
        carrier_details: Jet Logistics

5. Email: Just a reminder about our meeting the day after at 10 AM.
   Extracted Information:
        origin: null,
        destination: null,
        expected_shipment_datetime: 0000-00-00 10:00:0000,
        types_of_service: null,
        warehouse: null,
        description: Just a reminder about our meeting the day after at 10 AM.,
        quantities: null,
        carrier_details: null

Output: {output_format}
"""


# Function to insert extracted shipment details into MySQL database
def insert_data(extracted_details):
    try:
        # Initialize MySQL database connection
        mydb = mysql.connector.connect(**DB_CONFIG)
        cursor = mydb.cursor()

        # Skip insertion if all required fields are empty
        required_fields = [
            'origin', 'destination', 'expected_shipment_datetime',
            'types_of_service', 'warehouse', 'description',
            'quantities', 'carrier_details'
        ]
        if all(extracted_details.get(field) in ["", None] for field in required_fields):
            print("Skipping insertion: All extracted values are empty.")
            return

        # Insert data into database
        sql = """
        INSERT INTO shipment_details (
            origin, destination, expected_shipment_datetime, types_of_service, 
            warehouse, description, quantities, carrier_details, 
            sender, receiver, cc, bcc, subject
        ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
        """
        values = (
            extracted_details.get('origin'),
            extracted_details.get('destination'),
            extracted_details.get('expected_shipment_datetime'),
            extracted_details.get('types_of_service'),
            extracted_details.get('warehouse'),
            extracted_details.get('description'),
            extracted_details.get('quantities'),
            extracted_details.get('carrier_details'),
            extracted_details.get('sender'),
            extracted_details.get('receiver'),
            extracted_details.get('cc'),
            extracted_details.get('bcc'),
            extracted_details.get('subject')
        )
        cursor.execute(sql, values)
        mydb.commit()
        print("Data inserted successfully.")

    except mysql.connector.Error as db_err:
        logger.error(f"Database error: {db_err}")
    except Exception as ex:
        logger.error(f"Error inserting data: {ex}")
        
llm = Llama(model_path='./Phi-3-mini-4k-instruct-gguf', n_ctx=2048)

def read_email():
    df = pd.read_csv('./emails.csv')
    for i in df['Body']:
        prompt_ = f"<s>system:{prompt}<|end|><s>user:{i}<|end|><s>assistant:"
        output = llm(prompt_, max_tokens=1000, temperature=0.1)

        print("*"*50)
        t = output['choices'][0]['text']
        extracted_details = json.loads(t[t.find('{'):t.find('}')+1].replace("'", '"'))
        # print(extracted_details)
        print(type(extracted_details))
        meta_data = {
            'sender': 'sender',
            'receiver': 'receiver',
            'cc': 'cc',
            'bcc': 'bcc',
            'subject': 'subject'
        }
        # print(type(meta_data))
        extracted_details.update(meta_data)
    
        print('full data about email ! ...::',extracted_details)
        insert_data(extracted_details)
    

# Global variables
running = False
loop_thread = None

# HTML content for the web interface
html_content = """
<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Email Processing</title>
    <style>
        body { font-family: Arial, sans-serif; margin: 50px; }
        h1 { color: #333; }
        button {
            padding: 10px 20px;
            margin: 10px;
            background-color: #4CAF50;
            color: white;
            border: none;
            cursor: pointer;
        }
        button.stop { background-color: #f44336; }
        #status { font-weight: bold; }
    </style>
    <script>
        async function startLoop() {
            const response = await fetch('/start', { method: 'POST' });
            const result = await response.text();
            document.getElementById("status").innerHTML = result;
        }

        async function stopLoop() {
            const response = await fetch('/stop', { method: 'POST' });
            const result = await response.text();
            document.getElementById("status").innerHTML = result;
        }
    </script>
</head>
<body>
    <h1>Email Processing Status: <span id="status">{{ status }}</span></h1>
    <button onclick="startLoop()">Start</button>
    <button class="stop" onclick="stopLoop()">Stop</button>
</body>
</html>
"""

# Function to process emails in a loop
def email_processing_loop():
    global running
    logging.info("Starting email processing loop...")
    data = read_email()
    print("$" * 100)  # Simulate email processing
    time.sleep(10)  # Check for new emails every 10 seconds
    

# Endpoint to display the current email processor status
@app.get("/", response_class=HTMLResponse)
async def home():
    global running
    status = "Running" if running else "Stopped"
    return HTMLResponse(content=html_content.replace("{{ status }}", status), status_code=200)

# Endpoint to start the email processing loop
@app.post("/start")
async def start_email_loop():
    global running, loop_thread
    if not running:
        running = True
        # loop_thread = threading.Thread(target=email_processing_loop, daemon=True)
        # loop_thread.start()
        logging.info("Email processing loop started.")
        email_processing_loop()
        return "Running"
    else:
        return "Already running"

# Endpoint to stop the email processing loop
@app.post("/stop")
async def stop_email_loop():
    global running
    if running:
        running = False
        logging.info("Email processing loop stopped.")
        return "Stopped"
    else:
        return "Already stopped"

if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO)
    logging.info("Starting FastAPI server...")
    import uvicorn
    uvicorn.run(app, host="0.0.0.0", port=8000)