Duplicate from dwancin/web-scraping

Co-authored-by: DW <[email protected]>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,13 @@
+---
+title: Web Scraping
+emoji: 🕵️
+colorFrom: red
+colorTo: red
+sdk: gradio
+sdk_version: 3.35.2
+app_file: app.py
+pinned: false
+duplicated_from: dwancin/web-scraping
+---
+
+https://huggingface.co/spaces/dwancin/web-scraping

app.py

@@ -0,0 +1,193 @@
+''' 
+# Web Scrapping 
+[@dwancin on HuggingFace](https://huggingface.co/spaces/dwancin/web-scraping)
+'''
+
+import os,re, requests, uuid, zipfile, hashlib, shutil
+import gradio as gr
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin, urlparse
+
+# Function to validate URLs
+def validator(url):
+    parsed = urlparse(url)
+    return bool(parsed.netloc) and bool(parsed.scheme)
+
+
+# Function to find files on webpage
+def finder(url, soup, media_type):
+    files = []
+
+    # find image files
+    if media_type == "image":
+        tags = ['jpg', 'jpeg', 'png', 'svg', 'gif', 'webp', 'tiff', 'psd', 'eps', 'ai', 'indd', 'raw']
+        for tag in soup.find_all('img'):
+            file = tag.get('src')
+            if any(tag in file for tag in tags):
+                file_url = file
+                if not validator(file_url):
+                    file_url = urljoin(url, file_url)
+                files.append(file_url)
+
+    # find text
+    elif media_type == "text":
+        text_tags = ['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'span', 'strong', 'pdf', 'txt', 'doc', 'rtf', 'docx']
+        for tag in text_tags:
+            for element in soup.find_all(tag):
+                files.append(element.get_text())
+
+    # find links
+    else:
+        for link in soup.find_all('a'):
+            file = link.get('href')
+            if media_type in file:
+                file_url = file
+                if not validator(file_url):
+                    file_url = urljoin(url, file_url)
+                files.append(file_url)
+
+    return files
+
+
+# Function to download the files
+def downloader(urls, folder_name):
+    os.makedirs(folder_name, exist_ok=True)
+    for i, url in enumerate(urls):
+        response = requests.get(url, stream=True)
+        file_extension = url.split(".")[-1].split("&")[0]
+        url_hash = hashlib.md5(url.encode()).hexdigest()
+        unique_id = str(uuid.uuid4())[:8]
+        file_name = f'{url_hash}-{unique_id}.{file_extension}'
+        file_name = file_name[:255]
+        file_name = re.sub(r'[\\/:"*?<>|]+', '_', file_name)
+        with open(f'{folder_name}/{file_name}', 'wb') as out_file:
+            out_file.write(response.content)
+        print(f"Downloaded file: {file_name}")
+
+
+# Function to create zip file
+def zipper(folder_name):
+    if os.listdir(folder_name):
+        with zipfile.ZipFile(f'{folder_name}.zip', 'w') as zipf:
+            for file in os.listdir(folder_name):
+                zipf.write(f'{folder_name}/{file}')
+        return f'{folder_name}.zip'
+    else:
+        return ""
+
+
+# Function to access website
+def scrapper(url, images=False, text=False):
+    try:
+        response = requests.get(url, timeout=10)
+        response.raise_for_status()
+    except (requests.exceptions.RequestException, ValueError):
+        raise gr.Error(f"Unable to access URL: {url}")
+        return None, None
+    soup = BeautifulSoup(response.content, 'html.parser')
+
+    # Clear all the previews folder data
+    if images:
+        shutil.rmtree('images', ignore_errors=True)
+    if text:
+        shutil.rmtree('text', ignore_errors=True)
+
+    # Add images to the image folder
+    if images:
+        image_urls = finder(url, soup, 'image')
+        os.makedirs('images', exist_ok=True)
+        if image_urls:
+            downloader(image_urls, 'images')
+        else:
+            raise gr.Error("Found no images.")
+
+    # Add text files to the text folder
+    if text:
+        text_content = finder(url, soup, 'text')
+        os.makedirs('text', exist_ok=True)
+        if text_content:
+            with open('text/content.txt', 'w') as text_file:
+                for line in text_content:
+                    text_file.write(line + '\n')
+
+    # Output folder(s) as zip files
+    images_zip_file, text_zip_file = None, None
+    if images and os.path.exists('images') and os.listdir('images'):
+        images_zip_file = zipper('images')
+    if text and os.path.exists('text') and os.listdir('text'):
+        text_zip_file = zipper('text')
+    return images_zip_file, text_zip_file
+
+
+# Function to find requests errors
+def checker(url, media_types):
+    if not url:
+        raise gr.Error("URL cannot be empty.")
+    if not url.startswith("https://"):
+        raise gr.Error("The URL must begin with https://")
+    if not media_types:
+        raise gr.Error("At least one media type must be selected.")
+    try:
+        image_file, text_file = scrapper(url, "Images" in media_types, "Text" in media_types)
+    except requests.exceptions.HTTPError as e:
+        if e.response.status_code == 403:
+            raise gr.Error("HTTP Error: Forbidden. Access to the URL is forbidden.")
+        else:
+            raise gr.Error(f"HTTP Error: {e.response.status_code}")
+    except TypeError as e:
+        raise gr.Error(f"TypeError: {str(e)}")
+    except (requests.exceptions.RequestException, ValueError):
+        raise gr.Error(f"Unable to access URL: {url}")
+    files = []
+    if "Text" in media_types and not text_file:
+        raise gr.Error("Found no text.")
+    if "Images" in media_types and not image_file:
+        raise gr.Error("Found no images.")
+    if image_file:
+        files.append(image_file)
+    if text_file:
+        files.append(text_file)
+
+    print(f"Returning downloaded files from {url} in {files} ...")
+
+    return files
+
+# Gradio Interface
+with gr.Blocks(theme="dwancin/theme") as app:
+    title = gr.Markdown('''# Web Scrapping 🕵️''')
+    description = gr.Markdown('''Get all media files from your desired webpages with just a few clicks.''')
+    with gr.Row():
+        with gr.Column(scale=0, min_width=480, variant="panel", elem_id="sd-panel"):
+            url_name = gr.Textbox(
+                placeholder="Enter URL here",
+                show_label=True,
+                label="Website",
+            )
+
+            media_types = gr.CheckboxGroup(
+                ["Images", "Text"],
+                value="Images",
+                label="Media types",
+            )
+
+            submit_button = gr.Button(
+                "Submit",
+                variant="primary",
+                interactive=True,
+            )
+
+        with gr.Column(scale=2):
+            output_files = gr.Files(
+                label="Output",
+                elem_id="file-list",
+                size="lg",
+                show_label=False,
+            )
+    
+    submit_button.click(
+        checker, 
+        inputs=[url_name, media_types], 
+        outputs=[output_files],
+    )
+
+app.launch()

requirements.txt

@@ -0,0 +1,12 @@
+gradio>=3.35.2
+beautifulsoup4>=4.12.2
+Pillow>=9.5.0
+requests>=2.31.0
+Flask>=2.3.2
+zipfile2>= 0.0.12
+urllib3>=2.0.3
+pytest-shutil>=1.7.0
+mime>=0.1.0
+mimetypes-extensions>=0.1.0
+uuid>=1.30
+pytube>=15.0.0