Spaces:
Sleeping
Sleeping
File size: 5,202 Bytes
ee99dd0 068d689 268f3a0 068d689 268f3a0 068d689 268f3a0 0a7a178 068d689 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 |
import os
import streamlit as st
import requests
from transformers import pipeline
import openai
from langchain import LLMChain, PromptTemplate
from langchain import HuggingFaceHub
from diffusers import StableDiffusionPipeline, EulerDiscreteScheduler
import torch
# Suppressing all warnings
import warnings
warnings.filterwarnings("ignore")
api_token = os.getenv('H_TOKEN')
# Image-to-text
def img2txt(url):
print("Initializing captioning model...")
captioning_model = pipeline("image-to-text", model="Salesforce/blip-image-captioning-base")
print("Generating text from the image...")
text = captioning_model(url, max_new_tokens=20)[0]["generated_text"]
print(text)
return text
# Text-to-story
model = "tiiuae/falcon-7b-instruct"
llm = HuggingFaceHub(
huggingfacehub_api_token = api_token,
repo_id = model,
verbose = False,
model_kwargs = {"temperature":0.2, "max_new_tokens": 4000})
def generate_story(scenario, llm):
template= """You are a story teller.
You get a scenario as an input text, and generates a short story out of it.
Context: {scenario}
Story:
"""
prompt = PromptTemplate(template=template, input_variables=["scenario"])
#Let's create our LLM chain now
chain = LLMChain(prompt=prompt, llm=llm)
story = chain.predict(scenario=scenario)
start_index = story.find("Story:") + len("Story:")
# Extract the text after "Story:"
story = story[start_index:].strip()
return story
# Text-to-speech
def txt2speech(text):
print("Initializing text-to-speech conversion...")
API_URL = "https://api-inference.huggingface.co/models/espnet/kan-bayashi_ljspeech_vits"
headers = {"Authorization": f"Bearer {api_token }"}
payloads = {'inputs': text}
response = requests.post(API_URL, headers=headers, json=payloads)
with open('audio_story.mp3', 'wb') as file:
file.write(response.content)
# text-to- image
def txt2img(text, style="realistic"):
model_id = "stabilityai/stable-diffusion-2"
# Use the Euler scheduler here instead
scheduler = EulerDiscreteScheduler.from_pretrained(model_id, subfolder="scheduler")
pipe = StableDiffusionPipeline.from_pretrained(model_id, scheduler=scheduler, torch_dtype=torch.float16)
pipe = pipe.to("cuda")
image = pipe(prompt = text, guidance_scale = 7.5).images[0]
return image
st.sidebar.title("Choose the task")
# Streamlit web app main function
def main():
with st.sidebar.expander("Audio Story"):
st.set_page_config(page_title="🎨 Image-to-Audio Story 🎧", page_icon="🖼️")
st.title("Turn the Image into Audio Story")
# Allows users to upload an image file
uploaded_file = st.file_uploader("# 📷 Upload an image...", type=["jpg", "jpeg", "png"])
# Parameters for LLM model (in the sidebar)
#st.sidebar.markdown("# LLM Inference Configuration Parameters")
#top_k = st.sidebar.number_input("Top-K", min_value=1, max_value=100, value=5)
#top_p = st.sidebar.number_input("Top-P", min_value=0.0, max_value=1.0, value=0.8)
#temperature = st.sidebar.number_input("Temperature", min_value=0.1, max_value=2.0, value=1.5)
if uploaded_file is not None:
# Reads and saves uploaded image file
bytes_data = uploaded_file.read()
with open("uploaded_image.jpg", "wb") as file:
file.write(bytes_data)
st.image(uploaded_file, caption='🖼️ Uploaded Image', use_column_width=True)
# Initiates AI processing and story generation
with st.spinner("## 🤖 AI is at Work! "):
scenario = img2txt("uploaded_image.jpg") # Extracts text from the image
story = generate_story(scenario, llm) # Generates a story based on the image text, LLM params
txt2speech(story) # Converts the story to audio
st.markdown("---")
st.markdown("## 📜 Image Caption")
st.write(scenario)
st.markdown("---")
st.markdown("## 📖 Story")
st.write(story)
st.markdown("---")
st.markdown("## 🎧 Audio Story")
st.audio("audio_story.mp3")
with st.sidebar.expander("Image Generator"):
st.title("Stable Diffusion Image Generation")
st.write("This app lets you generate images using Stable Diffusion with the Euler scheduler.")
prompt = st.text_input("Enter your prompt:")
image_style = st.selectbox("Style Selection", ["realistic", "cartoon", "watercolor"])
if st.button("Generate Image"):
if prompt:
with st.spinner("Generating image..."):
image = txt2img(prompt= prompt, style = image_style)
st.image(image)
else:
st.error("Please enter a prompt.")
st.title("Welcome to your Creative Canvas!")
st.write("Use the tools in the sidebar to create audio stories and unique images.")
if __name__ == '__main__':
main() |