import os import streamlit as st import requests from transformers import pipeline import openai from langchain import LLMChain, PromptTemplate from langchain import HuggingFaceHub from diffusers import StableDiffusionPipeline, EulerDiscreteScheduler import torch # Suppressing all warnings import warnings warnings.filterwarnings("ignore") api_token = os.getenv('H_TOKEN') # Image-to-text def img2txt(url): print("Initializing captioning model...") captioning_model = pipeline("image-to-text", model="Salesforce/blip-image-captioning-base") print("Generating text from the image...") text = captioning_model(url, max_new_tokens=20)[0]["generated_text"] print(text) return text # Text-to-story model = "tiiuae/falcon-7b-instruct" llm = HuggingFaceHub( huggingfacehub_api_token = api_token, repo_id = model, verbose = False, model_kwargs = {"temperature":0.2, "max_new_tokens": 4000}) def generate_story(scenario, llm): template= """You are a story teller. You get a scenario as an input text, and generates a short story out of it. Context: {scenario} Story: """ prompt = PromptTemplate(template=template, input_variables=["scenario"]) #Let's create our LLM chain now chain = LLMChain(prompt=prompt, llm=llm) story = chain.predict(scenario=scenario) start_index = story.find("Story:") + len("Story:") # Extract the text after "Story:" story = story[start_index:].strip() return story # Text-to-speech def txt2speech(text): print("Initializing text-to-speech conversion...") API_URL = "https://api-inference.huggingface.co/models/espnet/kan-bayashi_ljspeech_vits" headers = {"Authorization": f"Bearer {api_token }"} payloads = {'inputs': text} response = requests.post(API_URL, headers=headers, json=payloads) with open('audio_story.mp3', 'wb') as file: file.write(response.content) # text-to- image def txt2img(text, style="realistic"): model_id = "stabilityai/stable-diffusion-2" # Use the Euler scheduler here instead scheduler = EulerDiscreteScheduler.from_pretrained(model_id, subfolder="scheduler") pipe = StableDiffusionPipeline.from_pretrained(model_id, scheduler=scheduler, torch_dtype=torch.float16) pipe = pipe.to("cuda") image = pipe(prompt = text, guidance_scale = 7.5).images[0] return image st.sidebar.title("Choose the task") # Streamlit web app main function def main(): with st.sidebar.expander("Audio Story"): st.set_page_config(page_title="🎨 Image-to-Audio Story 🎧", page_icon="🖼️") st.title("Turn the Image into Audio Story") # Allows users to upload an image file uploaded_file = st.file_uploader("# 📷 Upload an image...", type=["jpg", "jpeg", "png"]) # Parameters for LLM model (in the sidebar) #st.sidebar.markdown("# LLM Inference Configuration Parameters") #top_k = st.sidebar.number_input("Top-K", min_value=1, max_value=100, value=5) #top_p = st.sidebar.number_input("Top-P", min_value=0.0, max_value=1.0, value=0.8) #temperature = st.sidebar.number_input("Temperature", min_value=0.1, max_value=2.0, value=1.5) if uploaded_file is not None: # Reads and saves uploaded image file bytes_data = uploaded_file.read() with open("uploaded_image.jpg", "wb") as file: file.write(bytes_data) st.image(uploaded_file, caption='🖼️ Uploaded Image', use_column_width=True) # Initiates AI processing and story generation with st.spinner("## 🤖 AI is at Work! "): scenario = img2txt("uploaded_image.jpg") # Extracts text from the image story = generate_story(scenario, llm) # Generates a story based on the image text, LLM params txt2speech(story) # Converts the story to audio st.markdown("---") st.markdown("## 📜 Image Caption") st.write(scenario) st.markdown("---") st.markdown("## 📖 Story") st.write(story) st.markdown("---") st.markdown("## 🎧 Audio Story") st.audio("audio_story.mp3") with st.sidebar.expander("Image Generator"): st.title("Stable Diffusion Image Generation") st.write("This app lets you generate images using Stable Diffusion with the Euler scheduler.") prompt = st.text_input("Enter your prompt:") image_style = st.selectbox("Style Selection", ["realistic", "cartoon", "watercolor"]) if st.button("Generate Image"): if prompt: with st.spinner("Generating image..."): image = txt2img(prompt= prompt, style = image_style) st.image(image) else: st.error("Please enter a prompt.") st.title("Welcome to your Creative Canvas!") st.write("Use the tools in the sidebar to create audio stories and unique images.") if __name__ == '__main__': main()