test

.env.txt

@@ -0,0 +1,4 @@
+GROQ_API_KEY = "YOUR_GROQ_API"
+OPENAI_API_KEY = "YOUR_OPENAI_API"
+ANTHROPIC_API_KEY = "YOUR_ANTHROPIC_API"
+# You can add more API keys here

.gitattributes

@@ -1,35 +1,12 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Documentation
+*.html linguist-documentation
+docs/* linguist-documentation
+docs/examples/* linguist-documentation
+docs/md_v2/* linguist-documentation
+
+# Explicitly mark Python as the main language
+*.py linguist-detectable=true
+*.py linguist-language=Python
+
+# Exclude HTML from language statistics
+*.html linguist-detectable=false

.gitignore

@@ -0,0 +1,232 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+Crawl4AI.egg-info/
+Crawl4AI.egg-info/*
+crawler_data.db
+.vscode/
+.tests/
+.test_pads/
+test_pad.py
+test_pad*.py
+.data/
+Crawl4AI.egg-info/
+
+requirements0.txt
+a.txt
+
+*.sh
+.idea
+docs/examples/.chainlit/
+docs/examples/.chainlit/*
+.chainlit/config.toml
+.chainlit/translations/en-US.json
+
+local/
+.files/
+
+a.txt
+.lambda_function.py
+ec2*
+
+update_changelog.sh
+
+.DS_Store
+docs/.DS_Store
+tmp/
+test_env/
+**/.DS_Store
+**/.DS_Store
+
+todo.md
+todo_executor.md
+git_changes.py
+git_changes.md
+pypi_build.sh
+git_issues.py
+git_issues.md
+
+.next/
+.tests/
+# .issues/
+.docs/
+.issues/
+.gitboss/
+todo_executor.md
+protect-all-except-feature.sh
+manage-collab.sh
+publish.sh
+combine.sh
+combined_output.txt
+.local
+.scripts
+tree.md
+tree.md
+.scripts
+.local
+.do
+/plans
+plans/
+
+# Codeium
+.codeiumignore

CHANGELOG.md

@@ -0,0 +1,1089 @@
+# Changelog
+
+All notable changes to Crawl4AI will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+---
+
+## [0.4.267] - 2025 - 01 - 06
+
+### Added
+- **Windows Event Loop Configuration**: Introduced a utility function `configure_windows_event_loop` to resolve `NotImplementedError` for asyncio subprocesses on Windows. ([#utils.py](crawl4ai/utils.py), [#tutorials/async-webcrawler-basics.md](docs/md_v3/tutorials/async-webcrawler-basics.md))
+- **`page_need_scroll` Method**: Added a method to determine if a page requires scrolling before taking actions in `AsyncPlaywrightCrawlerStrategy`. ([#async_crawler_strategy.py](crawl4ai/async_crawler_strategy.py))
+
+### Changed
+- **Version Bump**: Updated the version from `0.4.246` to `0.4.247`. ([#__version__.py](crawl4ai/__version__.py))
+- **Improved Scrolling Logic**: Enhanced scrolling methods in `AsyncPlaywrightCrawlerStrategy` by adding a `scroll_delay` parameter for better control. ([#async_crawler_strategy.py](crawl4ai/async_crawler_strategy.py))
+- **Markdown Generation Example**: Updated the `hello_world.py` example to reflect the latest API changes and better illustrate features. ([#examples/hello_world.py](docs/examples/hello_world.py))
+- **Documentation Update**: 
+  - Added Windows-specific instructions for handling asyncio event loops. ([#async-webcrawler-basics.md](docs/md_v3/tutorials/async-webcrawler-basics.md))
+
+### Removed
+- **Legacy Markdown Generation Code**: Removed outdated and unused code for markdown generation in `content_scraping_strategy.py`. ([#content_scraping_strategy.py](crawl4ai/content_scraping_strategy.py))
+
+### Fixed
+- **Page Closing to Prevent Memory Leaks**:
+  - **Description**: Added a `finally` block to ensure pages are closed when no `session_id` is provided.
+  - **Impact**: Prevents memory leaks caused by lingering pages after a crawl.
+  - **File**: [`async_crawler_strategy.py`](crawl4ai/async_crawler_strategy.py)
+  - **Code**:
+    ```python
+    finally:
+        # If no session_id is given we should close the page
+        if not config.session_id:
+            await page.close()
+    ```
+- **Multiple Element Selection**: Modified `_get_elements` in `JsonCssExtractionStrategy` to return all matching elements instead of just the first one, ensuring comprehensive extraction. ([#extraction_strategy.py](crawl4ai/extraction_strategy.py))
+- **Error Handling in Scrolling**: Added robust error handling to ensure scrolling proceeds safely even if a configuration is missing. ([#async_crawler_strategy.py](crawl4ai/async_crawler_strategy.py))
+
+### Other
+- **Git Ignore Update**: Added `/plans` to `.gitignore` for better development environment consistency. ([#.gitignore](.gitignore))
+
+
+## [0.4.24] - 2024-12-31
+
+### Added
+- **Browser and SSL Handling**
+  - SSL certificate validation options in extraction strategies
+  - Custom certificate paths support
+  - Configurable certificate validation skipping
+  - Enhanced response status code handling with retry logic
+
+- **Content Processing**
+  - New content filtering system with regex support
+  - Advanced chunking strategies for large content
+  - Memory-efficient parallel processing
+  - Configurable chunk size optimization
+
+- **JSON Extraction**
+  - Complex JSONPath expression support
+  - JSON-CSS and Microdata extraction
+  - RDFa parsing capabilities
+  - Advanced data transformation pipeline
+
+- **Field Types**
+  - New field types: `computed`, `conditional`, `aggregate`, `template`
+  - Field inheritance system
+  - Reusable field definitions
+  - Custom validation rules
+
+### Changed
+- **Performance**
+  - Optimized selector compilation with caching
+  - Improved HTML parsing efficiency
+  - Enhanced memory management for large documents
+  - Batch processing optimizations
+
+- **Error Handling**
+  - More detailed error messages and categorization
+  - Enhanced debugging capabilities
+  - Improved performance metrics tracking
+  - Better error recovery mechanisms
+
+### Deprecated
+- Old field computation method using `eval`
+- Direct browser manipulation without proper SSL handling
+- Simple text-based content filtering
+
+### Removed
+- Legacy extraction patterns without proper error handling
+- Unsafe eval-based field computation
+- Direct DOM manipulation without sanitization
+
+### Fixed
+- Memory leaks in large document processing
+- SSL certificate validation issues
+- Incorrect handling of nested JSON structures
+- Performance bottlenecks in parallel processing
+
+### Security
+- Improved input validation and sanitization
+- Safe expression evaluation system
+- Enhanced resource protection
+- Rate limiting implementation
+
+## [0.4.1] - 2024-12-08
+
+### **File: `crawl4ai/async_crawler_strategy.py`**
+
+#### **New Parameters and Attributes Added**
+- **`text_mode` (boolean)**: Enables text-only mode, disables images, JavaScript, and GPU-related features for faster, minimal rendering.
+- **`light_mode` (boolean)**: Optimizes the browser by disabling unnecessary background processes and features for efficiency.
+- **`viewport_width` and `viewport_height`**: Dynamically adjusts based on `text_mode` mode (default values: 800x600 for `text_mode`, 1920x1080 otherwise).
+- **`extra_args`**: Adds browser-specific flags for `text_mode` mode.
+- **`adjust_viewport_to_content`**: Dynamically adjusts the viewport to the content size for accurate rendering.
+
+#### **Browser Context Adjustments**
+- Added **`viewport` adjustments**: Dynamically computed based on `text_mode` or custom configuration.
+- Enhanced support for `light_mode` and `text_mode` by adding specific browser arguments to reduce resource consumption.
+
+#### **Dynamic Content Handling**
+- **Full Page Scan Feature**:
+  - Scrolls through the entire page while dynamically detecting content changes.
+  - Ensures scrolling stops when no new dynamic content is loaded.
+
+#### **Session Management**
+- Added **`create_session`** method:
+  - Creates a new browser session and assigns a unique ID.
+  - Supports persistent and non-persistent contexts with full compatibility for cookies, headers, and proxies.
+
+#### **Improved Content Loading and Adjustment**
+- **`adjust_viewport_to_content`**:
+  - Automatically adjusts viewport to match content dimensions.
+  - Includes scaling via Chrome DevTools Protocol (CDP).
+- Enhanced content loading:
+  - Waits for images to load and ensures network activity is idle before proceeding.
+
+#### **Error Handling and Logging**
+- Improved error handling and detailed logging for:
+  - Viewport adjustment (`adjust_viewport_to_content`).
+  - Full page scanning (`scan_full_page`).
+  - Dynamic content loading.
+
+#### **Refactoring and Cleanup**
+- Removed hardcoded viewport dimensions in multiple places, replaced with dynamic values (`self.viewport_width`, `self.viewport_height`).
+- Removed commented-out and unused code for better readability.
+- Added default value for `delay_before_return_html` parameter.
+
+#### **Optimizations**
+- Reduced resource usage in `light_mode` by disabling unnecessary browser features such as extensions, background timers, and sync.
+- Improved compatibility for different browser types (`chrome`, `firefox`, `webkit`).
+
+---
+
+### **File: `docs/examples/quickstart_async.py`**
+
+#### **Schema Adjustment**
+- Changed schema reference for `LLMExtractionStrategy`:
+  - **Old**: `OpenAIModelFee.schema()`
+  - **New**: `OpenAIModelFee.model_json_schema()`
+  - This likely ensures better compatibility with the `OpenAIModelFee` class and its JSON schema.
+
+#### **Documentation Comments Updated**
+- Improved extraction instruction for schema-based LLM strategies.
+
+---
+
+### **New Features Added**
+1. **Text-Only Mode**:
+   - Focuses on minimal resource usage by disabling non-essential browser features.
+2. **Light Mode**:
+   - Optimizes browser for performance by disabling background tasks and unnecessary services.
+3. **Full Page Scanning**:
+   - Ensures the entire content of a page is crawled, including dynamic elements loaded during scrolling.
+4. **Dynamic Viewport Adjustment**:
+   - Automatically resizes the viewport to match content dimensions, improving compatibility and rendering accuracy.
+5. **Session Management**:
+   - Simplifies session handling with better support for persistent and non-persistent contexts.
+
+---
+
+### **Bug Fixes**
+- Fixed potential viewport mismatches by ensuring consistent use of `self.viewport_width` and `self.viewport_height` throughout the code.
+- Improved robustness of dynamic content loading to avoid timeouts and failed evaluations.
+
+
+
+
+
+
+
+## [0.3.75] December 1, 2024
+
+### PruningContentFilter
+
+#### 1. Introduced PruningContentFilter (Dec 01, 2024) (Dec 01, 2024)
+A new content filtering strategy that removes less relevant nodes based on metrics like text and link density.
+
+**Affected Files:**
+- `crawl4ai/content_filter_strategy.py`: Enhancement of content filtering capabilities.
+```diff
+Implemented effective pruning algorithm with comprehensive scoring.
+```
+- `README.md`: Improved documentation regarding new features.
+```diff
+Updated to include usage and explanation for the PruningContentFilter.
+```
+- `docs/md_v2/basic/content_filtering.md`: Expanded documentation for users.
+```diff
+Added detailed section explaining the PruningContentFilter.
+```
+
+#### 2. Added Unit Tests for PruningContentFilter (Dec 01, 2024) (Dec 01, 2024)
+Comprehensive tests added to ensure correct functionality of PruningContentFilter
+
+**Affected Files:**
+- `tests/async/test_content_filter_prune.py`: Increased test coverage for content filtering strategies.
+```diff
+Created test cases for various scenarios using the PruningContentFilter.
+```
+
+### Development Updates
+
+#### 3. Enhanced BM25ContentFilter tests (Dec 01, 2024) (Dec 01, 2024)
+Extended testing to cover additional edge cases and performance metrics.
+
+**Affected Files:**
+- `tests/async/test_content_filter_bm25.py`: Improved reliability and performance assurance.
+```diff
+Added tests for new extraction scenarios including malformed HTML.
+```
+
+### Infrastructure & Documentation
+
+#### 4. Updated Examples (Dec 01, 2024) (Dec 01, 2024)
+Altered examples in documentation to promote the use of PruningContentFilter alongside existing strategies.
+
+**Affected Files:**
+- `docs/examples/quickstart_async.py`: Enhanced usability and clarity for new users.
+- Revised example to illustrate usage of PruningContentFilter.
+
+## [0.3.746] November 29, 2024
+
+### Major Features
+1. Enhanced Docker Support (Nov 29, 2024)
+   - Improved GPU support in Docker images.
+   - Dockerfile refactored for better platform-specific installations.
+   - Introduced new Docker commands for different platforms:
+     - `basic-amd64`, `all-amd64`, `gpu-amd64` for AMD64.
+     - `basic-arm64`, `all-arm64`, `gpu-arm64` for ARM64.
+
+### Infrastructure & Documentation
+- Enhanced README.md to improve user guidance and installation instructions.
+- Added installation instructions for Playwright setup in README.
+- Created and updated examples in `docs/examples/quickstart_async.py` to be more useful and user-friendly.
+- Updated `requirements.txt` with a new `pydantic` dependency.
+- Bumped version number in `crawl4ai/__version__.py` to 0.3.746.
+
+### Breaking Changes
+- Streamlined application structure:
+  - Removed static pages and related code from `main.py` which might affect existing deployments relying on static content.
+
+### Development Updates
+- Developed `post_install` method in `crawl4ai/install.py` to streamline post-installation setup tasks.
+- Refined migration processes in `crawl4ai/migrations.py` with enhanced logging for better error visibility.
+- Updated `docker-compose.yml` to support local and hub services for different architectures, enhancing build and deploy capabilities.
+- Refactored example test cases in `docs/examples/docker_example.py` to facilitate comprehensive testing.
+
+### README.md
+Updated README with new docker commands and setup instructions.
+Enhanced installation instructions and guidance.
+
+### crawl4ai/install.py
+Added post-install script functionality.
+Introduced `post_install` method for automation of post-installation tasks.
+
+### crawl4ai/migrations.py
+Improved migration logging.
+Refined migration processes and added better logging.
+
+### docker-compose.yml
+Refactored docker-compose for better service management.
+Updated to define services for different platforms and versions.
+
+### requirements.txt
+Updated dependencies.
+Added `pydantic` to requirements file.
+
+### crawler/__version__.py
+Updated version number.
+Bumped version number to 0.3.746.
+
+### docs/examples/quickstart_async.py
+Enhanced example scripts.
+Uncommented example usage in async guide for user functionality.
+
+### main.py
+Refactored code to improve maintainability.
+Streamlined app structure by removing static pages code.
+
+## [0.3.743] November 27, 2024
+
+Enhance features and documentation
+- Updated version to 0.3.743
+- Improved ManagedBrowser configuration with dynamic host/port
+- Implemented fast HTML formatting in web crawler
+- Enhanced markdown generation with a new generator class
+- Improved sanitization and utility functions
+- Added contributor details and pull request acknowledgments
+- Updated documentation for clearer usage scenarios
+- Adjusted tests to reflect class name changes
+
+### CONTRIBUTORS.md
+Added new contributors and pull request details.
+Updated community contributions and acknowledged pull requests.
+
+### crawl4ai/__version__.py
+Version update.
+Bumped version to 0.3.743.
+
+### crawl4ai/async_crawler_strategy.py
+Improved ManagedBrowser configuration.
+Enhanced browser initialization with configurable host and debugging port; improved hook execution.
+
+### crawl4ai/async_webcrawler.py
+Optimized HTML processing.
+Implemented 'fast_format_html' for optimized HTML formatting; applied it when 'prettiify' is enabled.
+
+### crawl4ai/content_scraping_strategy.py
+Enhanced markdown generation strategy.
+Updated to use DefaultMarkdownGenerator and improved markdown generation with filters option.
+
+### crawl4ai/markdown_generation_strategy.py
+Refactored markdown generation class.
+Renamed DefaultMarkdownGenerationStrategy to DefaultMarkdownGenerator; added content filter handling.
+
+### crawl4ai/utils.py
+Enhanced utility functions.
+Improved input sanitization and enhanced HTML formatting method.
+
+### docs/md_v2/advanced/hooks-auth.md
+Improved documentation for hooks.
+Updated code examples to include cookies in crawler strategy initialization.
+
+### tests/async/test_markdown_genertor.py
+Refactored tests to match class renaming.
+Updated tests to use renamed DefaultMarkdownGenerator class.
+
+## [0.3.74] November 17, 2024
+
+This changelog details the updates and changes introduced in Crawl4AI version 0.3.74. It's designed to inform developers about new features, modifications to existing components, removals, and other important information.
+
+### 1. File Download Processing
+
+- Users can now specify download folders using the `downloads_path` parameter in the `AsyncWebCrawler` constructor or the `arun` method. If not specified, downloads are saved to a "downloads" folder within the `.crawl4ai` directory.
+- File download tracking is integrated into the `CrawlResult` object.  Successfully downloaded files are listed in the `downloaded_files` attribute, providing their paths.
+- Added `accept_downloads` parameter to the crawler strategies (defaults to `False`). If set to True you can add JS code and `wait_for` parameter for file download.
+
+**Example:**
+
+```python
+import asyncio
+import os
+from pathlib import Path
+from crawl4ai import AsyncWebCrawler
+
+async def download_example():
+    downloads_path = os.path.join(Path.home(), ".crawl4ai", "downloads")
+    os.makedirs(downloads_path, exist_ok=True)
+
+    async with AsyncWebCrawler(
+        accept_downloads=True, 
+        downloads_path=downloads_path, 
+        verbose=True
+    ) as crawler:
+        result = await crawler.arun(
+            url="https://www.python.org/downloads/",
+            js_code="""
+                const downloadLink = document.querySelector('a[href$=".exe"]');
+                if (downloadLink) { downloadLink.click(); }
+            """,
+            wait_for=5 # To ensure download has started
+        )
+
+        if result.downloaded_files:
+            print("Downloaded files:")
+            for file in result.downloaded_files:
+                print(f"- {file}")
+
+asyncio.run(download_example())
+
+```
+
+### 2. Refined Content Filtering
+
+- Introduced the `RelevanceContentFilter` strategy (and its implementation `BM25ContentFilter`) for extracting relevant content from web pages, replacing Fit Markdown and other content cleaning strategy. This new strategy leverages the BM25 algorithm to identify chunks of text relevant to the page's title, description, keywords, or a user-provided query.
+- The `fit_markdown` flag in the content scraper is used to filter content based on title, meta description, and keywords.
+
+**Example:**
+
+```python
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.content_filter_strategy import BM25ContentFilter
+
+async def filter_content(url, query):
+    async with AsyncWebCrawler() as crawler:
+        content_filter = BM25ContentFilter(user_query=query)
+        result = await crawler.arun(url=url, extraction_strategy=content_filter, fit_markdown=True)
+        print(result.extracted_content)  # Or result.fit_markdown for the markdown version
+        print(result.fit_html) # Or result.fit_html to show HTML with only the filtered content
+
+asyncio.run(filter_content("https://en.wikipedia.org/wiki/Apple", "fruit nutrition health"))
+```
+
+### 3. Raw HTML and Local File Support
+
+- Added support for crawling local files and raw HTML content directly.
+- Use the `file://` prefix for local file paths.
+- Use the `raw:` prefix for raw HTML strings.
+
+**Example:**
+
+```python
+async def crawl_local_or_raw(crawler, content, content_type):
+    prefix = "file://" if content_type == "local" else "raw:"
+    url = f"{prefix}{content}"
+    result = await crawler.arun(url=url)
+    if result.success:
+        print(f"Markdown Content from {content_type.title()} Source:")
+        print(result.markdown)
+
+# Example usage with local file and raw HTML
+async def main():
+    async with AsyncWebCrawler() as crawler:
+        # Local File
+        await crawl_local_or_raw(
+            crawler, os.path.abspath('tests/async/sample_wikipedia.html'), "local"
+        )
+        # Raw HTML
+        await crawl_raw_html(crawler, "<h1>Raw Test</h1><p>This is raw HTML.</p>")
+        
+
+asyncio.run(main())
+```
+
+### 4. Browser Management
+
+- New asynchronous crawler strategy implemented using Playwright.
+- `ManagedBrowser` class introduced for improved browser session handling, offering features like persistent browser sessions between requests (using  `session_id`  parameter) and browser process monitoring.
+- Updated to tf-playwright-stealth for enhanced stealth capabilities.
+- Added `use_managed_browser`, `use_persistent_context`, and `chrome_channel` parameters to AsyncPlaywrightCrawlerStrategy.
+
+
+**Example:**
+```python
+async def browser_management_demo():
+    user_data_dir = os.path.join(Path.home(), ".crawl4ai", "user-data-dir")
+    os.makedirs(user_data_dir, exist_ok=True)  # Ensure directory exists
+    async with AsyncWebCrawler(
+        use_managed_browser=True,
+        user_data_dir=user_data_dir,
+        use_persistent_context=True,
+        verbose=True
+    ) as crawler:
+        result1 = await crawler.arun(
+            url="https://example.com", session_id="my_session"
+        )
+        result2 = await crawler.arun(
+            url="https://example.com/anotherpage", session_id="my_session"
+        )
+
+asyncio.run(browser_management_demo())
+```
+
+
+### 5. API Server & Cache Improvements
+
+- Added CORS support to API server.
+- Implemented static file serving.
+- Enhanced root redirect functionality.
+- Cache database updated to store response headers and downloaded files information. It utilizes a file system approach to manage large content efficiently.
+- New, more efficient caching database built using xxhash and file system approach.
+- Introduced `CacheMode` enum (`ENABLED`, `DISABLED`, `READ_ONLY`, `WRITE_ONLY`, `BYPASS`) and `always_bypass_cache` parameter in AsyncWebCrawler for fine-grained cache control. This replaces `bypass_cache`, `no_cache_read`, `no_cache_write`, and `always_by_pass_cache`.
+
+
+### 🗑️ Removals
+
+- Removed deprecated: `crawl4ai/content_cleaning_strategy.py`.
+- Removed internal class ContentCleaningStrategy
+- Removed legacy cache control flags:  `bypass_cache`,  `disable_cache`,  `no_cache_read`,  `no_cache_write`, and `always_by_pass_cache`.  These have been superseded by  `cache_mode`.
+
+
+### ⚙️ Other Changes
+
+- Moved version file to `crawl4ai/__version__.py`.
+- Added `crawl4ai/cache_context.py`.
+- Added `crawl4ai/version_manager.py`.
+- Added `crawl4ai/migrations.py`.
+- Added `crawl4ai-migrate` entry point.
+- Added config `NEED_MIGRATION` and `SHOW_DEPRECATION_WARNINGS`.
+- API server now requires an API token for authentication, configurable with the `CRAWL4AI_API_TOKEN` environment variable.  This enhances API security.
+- Added synchronous crawl endpoint `/crawl_sync` for immediate result retrieval, and direct crawl endpoint `/crawl_direct` bypassing the task queue.
+
+
+### ⚠️ Deprecation Notices
+
+- The synchronous version of `WebCrawler` is being phased out.  While still available via `crawl4ai[sync]`, it will eventually be removed. Transition to `AsyncWebCrawler` is strongly recommended. Boolean cache control flags in `arun` are also deprecated, migrate to using the `cache_mode` parameter.  See examples in the "New Features" section above for correct usage.
+
+
+### 🐛 Bug Fixes
+
+- Resolved issue with browser context closing unexpectedly in Docker. This significantly improves stability, particularly within containerized environments. 
+- Fixed memory leaks associated with incorrect asynchronous cleanup by removing the `__del__` method and ensuring the browser context is closed explicitly using context managers.
+- Improved error handling in `WebScrapingStrategy`. More detailed error messages and suggestions for debugging will minimize frustration when running into unexpected issues.
+- Fixed issue with incorrect text parsing in specific HTML structures.
+
+
+### Example of migrating to the new CacheMode:
+
+**Old way:**
+
+```python
+crawler = AsyncWebCrawler(always_by_pass_cache=True)
+result = await crawler.arun(url="https://example.com", bypass_cache=True)
+```
+
+**New way:**
+
+```python
+from crawl4ai import CacheMode
+
+crawler = AsyncWebCrawler(always_bypass_cache=True)
+result = await crawler.arun(url="https://example.com", cache_mode=CacheMode.BYPASS)
+```
+
+
+## [0.3.74] - November 13, 2024
+
+1. **File Download Processing** (Nov 14, 2024)
+   - Added capability for users to specify download folders
+   - Implemented file download tracking in crowd result object
+   - Created new file: `tests/async/test_async_doanloader.py`
+
+2. **Content Filtering Improvements** (Nov 14, 2024)
+   - Introduced Relevance Content Filter as an improvement over Fit Markdown
+   - Implemented BM25 algorithm for content relevance matching
+   - Added new file: `crawl4ai/content_filter_strategy.py`
+   - Removed deprecated: `crawl4ai/content_cleaning_strategy.py`
+
+3. **Local File and Raw HTML Support** (Nov 13, 2024)
+   - Added support for processing local files
+   - Implemented raw HTML input handling in AsyncWebCrawler
+   - Enhanced `crawl4ai/async_webcrawler.py` with significant performance improvements
+
+4. **Browser Management Enhancements** (Nov 12, 2024)
+   - Implemented new async crawler strategy using Playwright
+   - Introduced ManagedBrowser for better browser session handling
+   - Added support for persistent browser sessions
+   - Updated from playwright_stealth to tf-playwright-stealth
+
+5. **API Server Component**
+   - Added CORS support
+   - Implemented static file serving
+   - Enhanced root redirect functionality
+
+
+
+## [0.3.731] - November 13, 2024
+
+### Added
+- Support for raw HTML and local file crawling via URL prefixes ('raw:', 'file://')
+- Browser process monitoring for managed browser instances
+- Screenshot capability for raw HTML and local file content
+- Response headers storage in cache database
+- New `fit_markdown` flag for optional markdown generation
+
+### Changed
+- Switched HTML parser from 'html.parser' to 'lxml' for ~4x performance improvement 
+- Optimized BeautifulSoup text conversion and element selection
+- Pre-compiled regular expressions for better performance
+- Improved metadata extraction efficiency
+- Response headers now stored alongside HTML in cache
+
+### Removed
+- `__del__` method from AsyncPlaywrightCrawlerStrategy to prevent async cleanup issues
+
+### Fixed 
+- Issue #256: Added support for crawling raw HTML content
+- Issue #253: Implemented file:// protocol handling
+- Missing response headers in cached results
+- Memory leaks from improper async cleanup
+
+## [v0.3.731] - 2024-11-13 Changelog for Issue 256 Fix
+- Fixed: Browser context unexpectedly closing in Docker environment during crawl operations.
+- Removed: __del__ method from AsyncPlaywrightCrawlerStrategy to prevent unreliable asynchronous cleanup, ensuring - browser context is closed explicitly within context managers.
+- Added: Monitoring for ManagedBrowser subprocess to detect and log unexpected terminations.
+- Updated: Dockerfile configurations to expose debugging port (9222) and allocate additional shared memory for improved browser stability.
+- Improved: Error handling and resource cleanup processes for browser lifecycle management within the Docker environment.
+
+## [v0.3.73] - 2024-11-05
+
+### Major Features
+- **New Doctor Feature**
+  - Added comprehensive system diagnostics tool
+  - Available through package hub and CLI
+  - Provides automated troubleshooting and system health checks
+  - Includes detailed reporting of configuration issues
+
+- **Dockerized API Server**
+  - Released complete Docker implementation for API server
+  - Added comprehensive documentation for Docker deployment
+  - Implemented container communication protocols
+  - Added environment configuration guides
+
+- **Managed Browser Integration**
+  - Added support for user-controlled browser instances
+  - Implemented `ManagedBrowser` class for better browser lifecycle management
+  - Added ability to connect to existing Chrome DevTools Protocol (CDP) endpoints
+  - Introduced user data directory support for persistent browser profiles
+
+- **Enhanced HTML Processing**
+  - Added HTML tag preservation feature during markdown conversion
+  - Introduced configurable tag preservation system
+  - Improved pre-tag and code block handling
+  - Added support for nested preserved tags with attribute retention
+
+### Improvements
+- **Browser Handling**
+  - Added flag to ignore body visibility for problematic pages
+  - Improved browser process cleanup and management
+  - Enhanced temporary directory handling for browser profiles
+  - Added configurable browser launch arguments
+
+- **Database Management**
+  - Implemented connection pooling for better performance
+  - Added retry logic for database operations
+  - Improved error handling and logging
+  - Enhanced cleanup procedures for database connections
+
+- **Resource Management**
+  - Added memory and CPU monitoring
+  - Implemented dynamic task slot allocation based on system resources
+  - Added configurable cleanup intervals
+
+### Technical Improvements
+- **Code Structure**
+  - Moved version management to dedicated _version.py file
+  - Improved error handling throughout the codebase
+  - Enhanced logging system with better error reporting
+  - Reorganized core components for better maintainability
+
+### Bug Fixes
+- Fixed issues with browser process termination
+- Improved handling of connection timeouts
+- Enhanced error recovery in database operations
+- Fixed memory leaks in long-running processes
+
+### Dependencies
+- Updated Playwright to v1.47
+- Updated core dependencies with more flexible version constraints
+- Added new development dependencies for testing
+
+### Breaking Changes
+- Changed default browser handling behavior
+- Modified database connection management approach
+- Updated API response structure for better consistency
+
+### Migration Guide
+When upgrading to v0.3.73, be aware of the following changes:
+
+1. Docker Deployment:
+   - Review Docker documentation for new deployment options
+   - Update environment configurations as needed
+   - Check container communication settings
+
+2. If using custom browser management:
+   - Update browser initialization code to use new ManagedBrowser class
+   - Review browser cleanup procedures
+
+3. For database operations:
+   - Check custom database queries for compatibility with new connection pooling
+   - Update error handling to work with new retry logic
+
+4. Using the Doctor:
+   - Run doctor command for system diagnostics: `crawl4ai doctor`
+   - Review generated reports for potential issues
+   - Follow recommended fixes for any identified problems
+
+
+## [v0.3.73] - 2024-11-04
+This commit introduces several key enhancements, including improved error handling and robust database operations in `async_database.py`, which now features a connection pool and retry logic for better reliability. Updates to the README.md provide clearer instructions and a better user experience with links to documentation sections. The `.gitignore` file has been refined to include additional directories, while the async web crawler now utilizes a managed browser for more efficient crawling. Furthermore, multiple dependency updates and introduction of the `CustomHTML2Text` class enhance text extraction capabilities.
+
+## [v0.3.73] - 2024-10-24
+
+### Added
+- preserve_tags: Added support for preserving specific HTML tags during markdown conversion.
+- Smart overlay removal system in AsyncPlaywrightCrawlerStrategy:
+  - Automatic removal of popups, modals, and cookie notices
+  - Detection and removal of fixed/sticky position elements
+  - Cleaning of empty block elements
+  - Configurable via `remove_overlay_elements` parameter
+- Enhanced screenshot capabilities:
+  - Added `screenshot_wait_for` parameter to control timing
+  - Improved screenshot handling with existing page context
+  - Better error handling with fallback error images
+- New URL normalization utilities:
+  - `normalize_url` function for consistent URL formatting
+  - `is_external_url` function for better link classification
+- Custom base directory support for cache storage:
+  - New `base_directory` parameter in AsyncWebCrawler
+  - Allows specifying alternative locations for `.crawl4ai` folder
+
+### Enhanced
+- Link handling improvements:
+  - Better duplicate link detection
+  - Enhanced internal/external link classification
+  - Improved handling of special URL protocols
+  - Support for anchor links and protocol-relative URLs
+- Configuration refinements:
+  - Streamlined social media domain list
+  - More focused external content filtering
+- LLM extraction strategy:
+  - Added support for separate API base URL via `api_base` parameter
+  - Better handling of base URLs in configuration
+
+### Fixed
+- Screenshot functionality:
+  - Resolved issues with screenshot timing and context
+  - Improved error handling and recovery
+- Link processing:
+  - Fixed URL normalization edge cases
+  - Better handling of invalid URLs
+  - Improved error messages for link processing failures
+
+### Developer Notes
+- The overlay removal system uses advanced JavaScript injection for better compatibility
+- URL normalization handles special cases like mailto:, tel:, and protocol-relative URLs
+- Screenshot system now reuses existing page context for better performance
+- Link processing maintains separate dictionaries for internal and external links to ensure uniqueness
+
+## [v0.3.72] - 2024-10-22
+
+### Added
+- New `ContentCleaningStrategy` class:
+  - Smart content extraction based on text density and element scoring
+  - Automatic removal of boilerplate content
+  - DOM tree analysis for better content identification
+  - Configurable thresholds for content detection
+- Advanced proxy support:
+  - Added `proxy_config` option for authenticated proxy connections
+  - Support for username/password in proxy configuration
+- New content output formats:
+  - `fit_markdown`: Optimized markdown output with main content focus
+  - `fit_html`: Clean HTML with only essential content
+
+### Enhanced
+- Image source detection:
+  - Support for multiple image source attributes (`src`, `data-src`, `srcset`, etc.)
+  - Automatic fallback through potential source attributes
+  - Smart handling of srcset attribute
+- External content handling:
+  - Made external link exclusion optional (disabled by default)
+  - Improved detection and handling of social media links
+  - Better control over external image filtering
+
+### Fixed
+- Image extraction reliability with multiple source attribute checks
+- External link and image handling logic for better accuracy
+
+### Developer Notes
+- The new `ContentCleaningStrategy` uses configurable thresholds for customization
+- Proxy configuration now supports more complex authentication scenarios
+- Content extraction process now provides both regular and optimized outputs
+
+## [v0.3.72] - 2024-10-20
+
+### Fixed
+- Added support for parsing Base64 encoded images in WebScrapingStrategy
+
+### Added
+- Forked and integrated a customized version of the html2text library for more control over Markdown generation
+- New configuration options for controlling external content:
+  - Ability to exclude all external links
+  - Option to specify domains to exclude (default includes major social media platforms)
+  - Control over excluding external images
+
+### Changed
+- Improved Markdown generation process:
+  - Added fine-grained control over character escaping in Markdown output
+  - Enhanced handling of code blocks and pre-formatted text
+- Updated `AsyncPlaywrightCrawlerStrategy.close()` method to use a shorter sleep time (0.5 seconds instead of 500)
+- Enhanced flexibility in `CosineStrategy` with a more generic `load_HF_embedding_model` function
+
+### Improved
+- Optimized content scraping and processing for better efficiency
+- Enhanced error handling and logging in various components
+
+### Developer Notes
+- The customized html2text library is now located within the crawl4ai package
+- New configuration options are available in the `config.py` file for external content handling
+- The `WebScrapingStrategy` class has been updated to accommodate new external content exclusion options
+
+## [v0.3.71] - 2024-10-19
+
+### Added
+- New chunking strategies:
+  - `OverlappingWindowChunking`: Allows for overlapping chunks of text, useful for maintaining context between chunks.
+  - Enhanced `SlidingWindowChunking`: Improved to handle edge cases and last chunks more effectively.
+
+### Changed
+- Updated `CHUNK_TOKEN_THRESHOLD` in config to 2048 tokens (2^11) for better compatibility with most LLM models.
+- Improved `AsyncPlaywrightCrawlerStrategy.close()` method to use a shorter sleep time (0.5 seconds instead of 500), significantly reducing wait time when closing the crawler.
+- Enhanced flexibility in `CosineStrategy`:
+  - Now uses a more generic `load_HF_embedding_model` function, allowing for easier swapping of embedding models.
+- Updated `JsonCssExtractionStrategy` and `JsonXPathExtractionStrategy` for better JSON-based extraction.
+
+### Fixed
+- Addressed potential issues with the sliding window chunking strategy to ensure all text is properly chunked.
+
+### Developer Notes
+- Added more comprehensive docstrings to chunking strategies for better code documentation.
+- Removed hardcoded device setting in `CosineStrategy`, now using the automatically detected device.
+- Added a new example in `quickstart_async.py` for generating a knowledge graph from crawled content.
+
+These updates aim to provide more flexibility in text processing, improve performance, and enhance the overall capabilities of the crawl4ai library. The new chunking strategies, in particular, offer more options for handling large texts in various scenarios.
+
+## [v0.3.71] - 2024-10-18
+
+### Changes
+1. **Version Update**:
+   - Updated version number from 0.3.7 to 0.3.71.
+
+2. **Crawler Enhancements**:
+   - Added `sleep_on_close` option to AsyncPlaywrightCrawlerStrategy for delayed browser closure.
+   - Improved context creation with additional options:
+     - Enabled `accept_downloads` and `java_script_enabled`.
+     - Added a cookie to enable cookies by default.
+
+3. **Error Handling Improvements**:
+   - Enhanced error messages in AsyncWebCrawler's `arun` method.
+   - Updated error reporting format for better visibility and consistency.
+
+4. **Performance Optimization**:
+   - Commented out automatic page and context closure in `crawl` method to potentially improve performance in certain scenarios.
+
+### Documentation
+- Updated quickstart notebook:
+  - Changed installation command to use the released package instead of GitHub repository.
+  - Updated kernel display name.
+
+### Developer Notes
+- Minor code refactoring and cleanup.
+
+## [v0.3.7] - 2024-10-17
+
+### New Features
+1. **Enhanced Browser Stealth**: 
+   - Implemented `playwright_stealth` for improved bot detection avoidance.
+   - Added `StealthConfig` for fine-tuned control over stealth parameters.
+
+2. **User Simulation**:
+   - New `simulate_user` option to mimic human-like interactions (mouse movements, clicks, keyboard presses).
+
+3. **Navigator Override**:
+   - Added `override_navigator` option to modify navigator properties, further improving bot detection evasion.
+
+4. **Improved iframe Handling**:
+   - New `process_iframes` parameter to extract and integrate iframe content into the main page.
+
+5. **Flexible Browser Selection**:
+   - Support for choosing between Chromium, Firefox, and WebKit browsers.
+
+6. **Include Links in Markdown**:
+    - Added support for including links in Markdown content, by definin g a new flag `include_links_on_markdown` in `crawl` method.   
+
+### Improvements
+1. **Better Error Handling**:
+   - Enhanced error reporting in WebScrapingStrategy with detailed error messages and suggestions.
+   - Added console message and error logging for better debugging.
+
+2. **Image Processing Enhancements**:
+   - Improved image dimension updating and filtering logic.
+
+3. **Crawling Flexibility**:
+   - Added support for custom viewport sizes.
+   - Implemented delayed content retrieval with `delay_before_return_html` parameter.
+
+4. **Performance Optimization**:
+   - Adjusted default semaphore count for parallel crawling.
+
+### Bug Fixes
+- Fixed an issue where the HTML content could be empty after processing.
+
+### Examples
+- Added new example `crawl_with_user_simulation()` demonstrating the use of user simulation and navigator override features.
+
+### Developer Notes
+- Refactored code for better maintainability and readability.
+- Updated browser launch arguments for improved compatibility and performance.
+
+## [v0.3.6] - 2024-10-12 
+
+### 1. Improved Crawling Control
+- **New Hook**: Added `before_retrieve_html` hook in `AsyncPlaywrightCrawlerStrategy`.
+- **Delayed HTML Retrieval**: Introduced `delay_before_return_html` parameter to allow waiting before retrieving HTML content.
+  - Useful for pages with delayed content loading.
+- **Flexible Timeout**: `smart_wait` function now uses `page_timeout` (default 60 seconds) instead of a fixed 30-second timeout.
+  - Provides better handling for slow-loading pages.
+- **How to use**: Set `page_timeout=your_desired_timeout` (in milliseconds) when calling `crawler.arun()`.
+
+### 2. Browser Type Selection
+- Added support for different browser types (Chromium, Firefox, WebKit).
+- Users can now specify the browser type when initializing AsyncWebCrawler.
+- **How to use**: Set `browser_type="firefox"` or `browser_type="webkit"` when initializing AsyncWebCrawler.
+
+### 3. Screenshot Capture
+- Added ability to capture screenshots during crawling.
+- Useful for debugging and content verification.
+- **How to use**: Set `screenshot=True` when calling `crawler.arun()`.
+
+### 4. Enhanced LLM Extraction Strategy
+- Added support for multiple LLM providers (OpenAI, Hugging Face, Ollama).
+- **Custom Arguments**: Added support for passing extra arguments to LLM providers via `extra_args` parameter.
+- **Custom Headers**: Users can now pass custom headers to the extraction strategy.
+- **How to use**: Specify the desired provider and custom arguments when using `LLMExtractionStrategy`.
+
+### 5. iframe Content Extraction
+- New feature to process and extract content from iframes.
+- **How to use**: Set `process_iframes=True` in the crawl method.
+
+### 6. Delayed Content Retrieval
+- Introduced `get_delayed_content` method in `AsyncCrawlResponse`.
+- Allows retrieval of content after a specified delay, useful for dynamically loaded content.
+- **How to use**: Access `result.get_delayed_content(delay_in_seconds)` after crawling.
+
+### Improvements and Optimizations
+
+#### 1. AsyncWebCrawler Enhancements
+- **Flexible Initialization**: Now accepts arbitrary keyword arguments, passed directly to the crawler strategy.
+- Allows for more customized setups.
+
+#### 2. Image Processing Optimization
+- Enhanced image handling in WebScrapingStrategy.
+- Added filtering for small, invisible, or irrelevant images.
+- Improved image scoring system for better content relevance.
+- Implemented JavaScript-based image dimension updating for more accurate representation.
+
+#### 3. Database Schema Auto-updates
+- Automatic database schema updates ensure compatibility with the latest version.
+
+#### 4. Enhanced Error Handling and Logging
+- Improved error messages and logging for easier debugging.
+
+#### 5. Content Extraction Refinements
+- Refined HTML sanitization process.
+- Improved handling of base64 encoded images.
+- Enhanced Markdown conversion process.
+- Optimized content extraction algorithms.
+
+#### 6. Utility Function Enhancements
+- `perform_completion_with_backoff` function now supports additional arguments for more customized API calls to LLM providers.
+
+### Bug Fixes
+- Fixed an issue where image tags were being prematurely removed during content extraction.
+
+### Examples and Documentation
+- Updated `quickstart_async.py` with examples of:
+  - Using custom headers in LLM extraction.
+  - Different LLM provider usage (OpenAI, Hugging Face, Ollama).
+  - Custom browser type usage.
+
+### Developer Notes
+- Refactored code for better maintainability, flexibility, and performance.
+- Enhanced type hinting throughout the codebase for improved development experience.
+- Expanded error handling for more robust operation.
+
+These updates significantly enhance the flexibility, accuracy, and robustness of crawl4ai, providing users with more control and options for their web crawling and content extraction tasks.
+
+## [v0.3.5] - 2024-09-02
+
+Enhance AsyncWebCrawler with smart waiting and screenshot capabilities
+
+- Implement smart_wait function in AsyncPlaywrightCrawlerStrategy
+- Add screenshot support to AsyncCrawlResponse and AsyncWebCrawler
+- Improve error handling and timeout management in crawling process
+- Fix typo in CrawlResult model (responser_headers -> response_headers)
+
+## [v0.2.77] - 2024-08-04
+
+Significant improvements in text processing and performance:
+
+- 🚀 **Dependency reduction**: Removed dependency on spaCy model for text chunk labeling in cosine extraction strategy.
+- 🤖 **Transformer upgrade**: Implemented text sequence classification using a transformer model for labeling text chunks.
+- ⚡ **Performance enhancement**: Improved model loading speed due to removal of spaCy dependency.
+- 🔧 **Future-proofing**: Laid groundwork for potential complete removal of spaCy dependency in future versions.
+
+These changes address issue #68 and provide a foundation for faster, more efficient text processing in Crawl4AI.
+
+## [v0.2.76] - 2024-08-02
+
+Major improvements in functionality, performance, and cross-platform compatibility! 🚀
+
+- 🐳 **Docker enhancements**: Significantly improved Dockerfile for easy installation on Linux, Mac, and Windows.
+- 🌐 **Official Docker Hub image**: Launched our first official image on Docker Hub for streamlined deployment.
+- 🔧 **Selenium upgrade**: Removed dependency on ChromeDriver, now using Selenium's built-in capabilities for better compatibility.
+- 🖼️ **Image description**: Implemented ability to generate textual descriptions for extracted images from web pages.
+- ⚡ **Performance boost**: Various improvements to enhance overall speed and performance.
+
+A big shoutout to our amazing community contributors:
+- [@aravindkarnam](https://github.com/aravindkarnam) for developing the textual description extraction feature.
+- [@FractalMind](https://github.com/FractalMind) for creating the first official Docker Hub image and fixing Dockerfile errors.
+- [@ketonkss4](https://github.com/ketonkss4) for identifying Selenium's new capabilities, helping us reduce dependencies.
+
+Your contributions are driving Crawl4AI forward! 🙌
+
+## [v0.2.75] - 2024-07-19
+
+Minor improvements for a more maintainable codebase:
+
+- 🔄 Fixed typos in `chunking_strategy.py` and `crawler_strategy.py` to improve code readability
+- 🔄 Removed `.test_pads/` directory from `.gitignore` to keep our repository clean and organized
+
+These changes may seem small, but they contribute to a more stable and sustainable codebase. By fixing typos and updating our `.gitignore` settings, we're ensuring that our code is easier to maintain and scale in the long run.
+
+## [v0.2.74] - 2024-07-08
+A slew of exciting updates to improve the crawler's stability and robustness! 🎉
+
+- 💻 **UTF encoding fix**: Resolved the Windows \"charmap\" error by adding UTF encoding.
+- 🛡️ **Error handling**: Implemented MaxRetryError exception handling in LocalSeleniumCrawlerStrategy.
+- 🧹 **Input sanitization**: Improved input sanitization and handled encoding issues in LLMExtractionStrategy.
+- 🚮 **Database cleanup**: Removed existing database file and initialized a new one.
+
+
+## [v0.2.73] - 2024-07-03
+
+💡 In this release, we've bumped the version to v0.2.73 and refreshed our documentation to ensure you have the best experience with our project.
+
+* Supporting website need "with-head" mode to crawl the website with head.
+* Fixing the installation issues for setup.py and dockerfile.
+* Resolve multiple issues.
+
+## [v0.2.72] - 2024-06-30
+
+This release brings exciting updates and improvements to our project! 🎉
+
+* 📚 **Documentation Updates**: Our documentation has been revamped to reflect the latest changes and additions.
+* 🚀 **New Modes in setup.py**: We've added support for three new modes in setup.py: default, torch, and transformers. This enhances the project's flexibility and usability.
+* 🐳 **Docker File Updates**: The Docker file has been updated to ensure seamless compatibility with the new modes and improvements.
+* 🕷️ **Temporary Solution for Headless Crawling**: We've implemented a temporary solution to overcome issues with crawling websites in headless mode.
+
+These changes aim to improve the overall user experience, provide more flexibility, and enhance the project's performance. We're thrilled to share these updates with you and look forward to continuing to evolve and improve our project!
+
+## [0.2.71] - 2024-06-26
+
+**Improved Error Handling and Performance** 🚧
+
+* 🚫 Refactored `crawler_strategy.py` to handle exceptions and provide better error messages, making it more robust and reliable.
+* 💻 Optimized the `get_content_of_website_optimized` function in `utils.py` for improved performance, reducing potential bottlenecks.
+* 💻 Updated `utils.py` with the latest changes, ensuring consistency and accuracy.
+* 🚫 Migrated to `ChromeDriverManager` to resolve Chrome driver download issues, providing a smoother user experience.
+
+These changes focus on refining the existing codebase, resulting in a more stable, efficient, and user-friendly experience. With these improvements, you can expect fewer errors and better performance in the crawler strategy and utility functions.
+
+## [0.2.71] - 2024-06-25
+### Fixed
+- Speed up twice the extraction function.
+
+
+## [0.2.6] - 2024-06-22
+### Fixed
+- Fix issue #19: Update Dockerfile to ensure compatibility across multiple platforms.
+
+## [0.2.5] - 2024-06-18
+### Added
+- Added five important hooks to the crawler:
+  - on_driver_created: Called when the driver is ready for initializations.
+  - before_get_url: Called right before Selenium fetches the URL.
+  - after_get_url: Called after Selenium fetches the URL.
+  - before_return_html: Called when the data is parsed and ready.
+  - on_user_agent_updated: Called when the user changes the user_agent, causing the driver to reinitialize.
+- Added an example in `quickstart.py` in the example folder under the docs.
+- Enhancement issue #24: Replaced inline HTML tags (e.g., DEL, INS, SUB, ABBR) with textual format for better context handling in LLM.
+- Maintaining the semantic context of inline tags (e.g., abbreviation, DEL, INS) for improved LLM-friendliness.
+- Updated Dockerfile to ensure compatibility across multiple platforms (Hopefully!).
+
+## [v0.2.4] - 2024-06-17
+### Fixed
+- Fix issue #22: Use MD5 hash for caching HTML files to handle long URLs

CODE_OF_CONDUCT.md

@@ -0,0 +1,131 @@
+# Crawl4AI Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, caste, color, religion, or sexual
+identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the overall
+  community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or advances of
+  any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email address,
+  without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official email address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+[email protected]. All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series of
+actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or permanent
+ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within the
+community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.1, available at
+[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
+
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
+[https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations

CONTRIBUTORS.md

@@ -0,0 +1,42 @@
+# Contributors to Crawl4AI
+
+We would like to thank the following people for their contributions to Crawl4AI:
+
+## Core Team
+
+- [Unclecode](https://github.com/unclecode) - Project Creator and Main Developer
+- [Nasrin](https://github.com/ntohidi) - Project Manager and Developer
+- [Aravind Karnam](https://github.com/aravindkarnam) - Developer
+
+## Community Contributors
+
+- [aadityakanjolia4](https://github.com/aadityakanjolia4) - Fix for `CustomHTML2Text` is not defined.
+- [FractalMind](https://github.com/FractalMind) - Created the first official Docker Hub image and fixed Dockerfile errors
+- [ketonkss4](https://github.com/ketonkss4) - Identified Selenium's new capabilities, helping reduce dependencies
+- [jonymusky](https://github.com/jonymusky) - Javascript execution documentation, and wait_for
+- [datehoer](https://github.com/datehoer) - Add browser prxy support
+
+## Pull Requests
+
+- [dvschuyl](https://github.com/dvschuyl) - AsyncPlaywrightCrawlerStrategy page-evaluate context destroyed by navigation [#304](https://github.com/unclecode/crawl4ai/pull/304)
+- [nelzomal](https://github.com/nelzomal) - Enhance development installation instructions [#286](https://github.com/unclecode/crawl4ai/pull/286)
+- [HamzaFarhan](https://github.com/HamzaFarhan) - Handled the cases where markdown_with_citations, references_markdown, and filtered_html might not be defined [#293](https://github.com/unclecode/crawl4ai/pull/293)
+- [NanmiCoder](https://github.com/NanmiCoder) - fix: crawler strategy exception handling and fixes [#271](https://github.com/unclecode/crawl4ai/pull/271)
+- [paulokuong](https://github.com/paulokuong) - fix: RAWL4_AI_BASE_DIRECTORY should be Path object instead of string [#298](https://github.com/unclecode/crawl4ai/pull/298)
+
+
+## Other Contributors
+
+- [Gokhan](https://github.com/gkhngyk) 
+- [Shiv Kumar](https://github.com/shivkumar0757)
+- [QIN2DIM](https://github.com/QIN2DIM)
+
+## Acknowledgements
+
+We also want to thank all the users who have reported bugs, suggested features, or helped in any other way to make Crawl4AI better.
+
+---
+
+If you've contributed to Crawl4AI and your name isn't on this list, please [open a pull request](https://github.com/unclecode/crawl4ai/pulls) with your name, link, and contribution, and we'll review it promptly.
+
+Thank you all for your contributions!

Dockerfile

@@ -0,0 +1,136 @@
+# syntax=docker/dockerfile:1.4
+
+ARG TARGETPLATFORM
+ARG BUILDPLATFORM
+
+# Other build arguments
+ARG PYTHON_VERSION=3.10
+
+# Base stage with system dependencies
+FROM python:${PYTHON_VERSION}-slim as base
+
+# Declare ARG variables again within the build stage
+ARG INSTALL_TYPE=basic
+ARG ENABLE_GPU=false
+
+# Platform-specific labels
+LABEL maintainer="unclecode"
+LABEL description="🔥🕷️ Crawl4AI: Open-source LLM Friendly Web Crawler & scraper"
+LABEL version="1.0"
+
+# Environment setup
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PIP_DEFAULT_TIMEOUT=100 \
+    DEBIAN_FRONTEND=noninteractive
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    curl \
+    wget \
+    gnupg \
+    git \
+    cmake \
+    pkg-config \
+    python3-dev \
+    libjpeg-dev \
+    libpng-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Playwright system dependencies for Linux
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libglib2.0-0 \
+    libnss3 \
+    libnspr4 \
+    libatk1.0-0 \
+    libatk-bridge2.0-0 \
+    libcups2 \
+    libdrm2 \
+    libdbus-1-3 \
+    libxcb1 \
+    libxkbcommon0 \
+    libx11-6 \
+    libxcomposite1 \
+    libxdamage1 \
+    libxext6 \
+    libxfixes3 \
+    libxrandr2 \
+    libgbm1 \
+    libpango-1.0-0 \
+    libcairo2 \
+    libasound2 \
+    libatspi2.0-0 \
+    && rm -rf /var/lib/apt/lists/*
+
+# GPU support if enabled and architecture is supported
+RUN if [ "$ENABLE_GPU" = "true" ] && [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \
+    apt-get update && apt-get install -y --no-install-recommends \
+    nvidia-cuda-toolkit \
+    && rm -rf /var/lib/apt/lists/* ; \
+else \
+    echo "Skipping NVIDIA CUDA Toolkit installation (unsupported platform or GPU disabled)"; \
+fi
+
+# Create and set working directory
+WORKDIR /app
+
+# Copy the entire project
+COPY . .
+
+# Install base requirements
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Install required library for FastAPI
+RUN pip install fastapi uvicorn psutil
+
+# Install ML dependencies first for better layer caching
+RUN if [ "$INSTALL_TYPE" = "all" ] ; then \
+        pip install --no-cache-dir \
+            torch \
+            torchvision \
+            torchaudio \
+            scikit-learn \
+            nltk \
+            transformers \
+            tokenizers && \
+        python -m nltk.downloader punkt stopwords ; \
+    fi
+
+# Install the package
+RUN if [ "$INSTALL_TYPE" = "all" ] ; then \
+        pip install ".[all]" && \
+        python -m crawl4ai.model_loader ; \
+    elif [ "$INSTALL_TYPE" = "torch" ] ; then \
+        pip install ".[torch]" ; \
+    elif [ "$INSTALL_TYPE" = "transformer" ] ; then \
+        pip install ".[transformer]" && \
+        python -m crawl4ai.model_loader ; \
+    else \
+        pip install "." ; \
+    fi
+
+    # Install MkDocs and required plugins
+RUN pip install --no-cache-dir \
+    mkdocs \
+    mkdocs-material \
+    mkdocs-terminal \
+    pymdown-extensions
+
+# Build MkDocs documentation
+RUN mkdocs build
+
+# Install Playwright and browsers
+RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then \
+    playwright install chromium; \
+    elif [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
+    playwright install chromium; \
+    fi
+
+# Expose port
+EXPOSE 8000 11235 9222 8080
+
+# Start the FastAPI server
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "11235"]

LICENSE

@@ -0,0 +1,51 @@
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document.
+
+"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License.
+
+"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
+
+"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License.
+
+"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files.
+
+"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types.
+
+"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below).
+
+"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof.
+
+"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution."
+
+"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions:
+
+You must give any other recipients of the Work or Derivative Works a copy of this License; and
+You must cause any modified files to carry prominent notices stating that You changed the files; and
+You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and
+If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License.
+You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License.
+
+5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS

MANIFEST.in

@@ -0,0 +1,2 @@
+include requirements.txt
+recursive-include crawl4ai/js_snippet *.js

MISSION.md

@@ -0,0 +1,46 @@
+# Mission
+
+![Mission Diagram](./docs/assets/pitch-dark.svg)
+
+### 1. The Data Capitalization Opportunity
+
+We live in an unprecedented era of digital wealth creation. Every day, individuals and enterprises generate massive amounts of valuable digital footprints across various platforms, social media channels, messenger apps, and cloud services. While people can interact with their data within these platforms, there's an immense untapped opportunity to transform this data into true capital assets. Just as physical property became a foundational element of wealth creation, personal and enterprise data has the potential to become a new form of capital on balance sheets.
+
+For individuals, this represents an opportunity to transform their digital activities into valuable assets. For enterprises, their internal communications, team discussions, and collaborative documents contain rich insights that could be structured and valued as intellectual capital. This wealth of information represents an unprecedented opportunity for value creation in the digital age.
+
+### 2. The Potential of Authentic Data
+
+While synthetic data has played a crucial role in AI development, there's an enormous untapped potential in the authentic data generated by individuals and organizations. Every message, document, and interaction contains unique insights and patterns that could enhance AI development. The challenge isn't a lack of data - it's that most authentic human-generated data remains inaccessible for productive use.
+
+By enabling willing participation in data sharing, we can unlock this vast reservoir of authentic human knowledge. This represents an opportunity to enhance AI development with diverse, real-world data that reflects the full spectrum of human experience and knowledge.
+
+## Our Pathway to Data Democracy
+
+### 1. Open-Source Foundation
+
+Our first step is creating an open-source data extraction engine that empowers developers and innovators to build tools for data structuring and organization. This foundation ensures transparency, security, and community-driven development. By making these tools openly available, we enable the technical infrastructure needed for true data ownership and capitalization.
+
+### 2. Data Capitalization Platform
+
+Building on this open-source foundation, we're developing a platform that helps individuals and enterprises transform their digital footprints into structured, valuable assets. This platform will provide the tools and frameworks needed to organize, understand, and value personal and organizational data as true capital assets.
+
+### 3. Creating a Data Marketplace
+
+The final piece is establishing a marketplace where individuals and organizations can willingly share their data assets. This creates opportunities for:
+- Individuals to earn equity, revenue, or other forms of value from their data
+- Enterprises to access diverse, high-quality data for AI development
+- Researchers to work with authentic human-generated data
+- Startups to build innovative solutions using real-world data
+
+## Economic Vision: A Shared Data Economy
+
+We envision a future where data becomes a fundamental asset class in a thriving shared economy. This transformation will democratize AI development by enabling willing participation in data sharing, ensuring that the benefits of AI advancement flow back to data creators. Just as property rights revolutionized economic systems, establishing data as a capital asset will create new opportunities for wealth creation and economic participation.
+
+This shared data economy will:
+- Enable individuals to capitalize on their digital footprints
+- Create new revenue streams for data creators
+- Provide AI developers with access to diverse, authentic data
+- Foster innovation through broader access to real-world data
+- Ensure more equitable distribution of AI's economic benefits
+
+Our vision is to facilitate this transformation from the ground up - starting with open-source tools, progressing to data capitalization platforms, and ultimately creating a thriving marketplace where data becomes a true asset class in a shared economy. This approach ensures that the future of AI is built on a foundation of authentic human knowledge, with benefits flowing back to the individuals and organizations who create and share their valuable data.

README.md

@@ -6,6 +6,565 @@ colorTo: pink
 sdk: docker
 pinned: false
 license: mit
+port: 11235
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+
+# 🚀🤖 Crawl4AI: Open-source LLM Friendly Web Crawler & Scraper.
+
+<div align="center">
+
+<a href="https://trendshift.io/repositories/11716" target="_blank"><img src="https://trendshift.io/api/badge/repositories/11716" alt="unclecode%2Fcrawl4ai | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+
+[![GitHub Stars](https://img.shields.io/github/stars/unclecode/crawl4ai?style=social)](https://github.com/unclecode/crawl4ai/stargazers)
+[![GitHub Forks](https://img.shields.io/github/forks/unclecode/crawl4ai?style=social)](https://github.com/unclecode/crawl4ai/network/members)
+
+[![PyPI version](https://badge.fury.io/py/crawl4ai.svg)](https://badge.fury.io/py/crawl4ai)
+[![Python Version](https://img.shields.io/pypi/pyversions/crawl4ai)](https://pypi.org/project/crawl4ai/)
+[![Downloads](https://static.pepy.tech/badge/crawl4ai/month)](https://pepy.tech/project/crawl4ai)
+
+<!-- [![Documentation Status](https://readthedocs.org/projects/crawl4ai/badge/?version=latest)](https://crawl4ai.readthedocs.io/) -->
+[![License](https://img.shields.io/github/license/unclecode/crawl4ai)](https://github.com/unclecode/crawl4ai/blob/main/LICENSE)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![Security: bandit](https://img.shields.io/badge/security-bandit-yellow.svg)](https://github.com/PyCQA/bandit)
+[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](code_of_conduct.md)
+
+</div>
+
+Crawl4AI is the #1 trending GitHub repository, actively maintained by a vibrant community. It delivers blazing-fast, AI-ready web crawling tailored for LLMs, AI agents, and data pipelines. Open source, flexible, and built for real-time performance, Crawl4AI empowers developers with unmatched speed, precision, and deployment ease.  
+
+[✨ Check out latest update v0.4.24x](#-recent-updates)
+
+🎉 **Version 0.4.24x is out!** Major improvements in extraction strategies with enhanced JSON handling, SSL security, and Amazon product extraction. Plus, a completely revamped content filtering system! [Read the release notes →](https://crawl4ai.com/mkdocs/blog)
+
+## 🧐 Why Crawl4AI?
+
+1. **Built for LLMs**: Creates smart, concise Markdown optimized for RAG and fine-tuning applications.  
+2. **Lightning Fast**: Delivers results 6x faster with real-time, cost-efficient performance.  
+3. **Flexible Browser Control**: Offers session management, proxies, and custom hooks for seamless data access.  
+4. **Heuristic Intelligence**: Uses advanced algorithms for efficient extraction, reducing reliance on costly models.  
+5. **Open Source & Deployable**: Fully open-source with no API keys—ready for Docker and cloud integration.  
+6. **Thriving Community**: Actively maintained by a vibrant community and the #1 trending GitHub repository.
+
+## 🚀 Quick Start 
+
+1. Install Crawl4AI:
+```bash
+# Install the package
+pip install -U crawl4ai
+
+# Run post-installation setup
+crawl4ai-setup
+
+# Verify your installation
+crawl4ai-doctor
+```
+
+If you encounter any browser-related issues, you can install them manually:
+```bash
+python -m playwright install --with-deps chromium
+```
+
+2. Run a simple web crawl:
+```python
+import asyncio
+from crawl4ai import *
+
+async def main():
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business",
+        )
+        print(result.markdown)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+## ✨ Features 
+
+<details>
+<summary>📝 <strong>Markdown Generation</strong></summary>
+
+- 🧹 **Clean Markdown**: Generates clean, structured Markdown with accurate formatting.
+- 🎯 **Fit Markdown**: Heuristic-based filtering to remove noise and irrelevant parts for AI-friendly processing.
+- 🔗 **Citations and References**: Converts page links into a numbered reference list with clean citations.
+- 🛠️ **Custom Strategies**: Users can create their own Markdown generation strategies tailored to specific needs.
+- 📚 **BM25 Algorithm**: Employs BM25-based filtering for extracting core information and removing irrelevant content. 
+</details>
+
+<details>
+<summary>📊 <strong>Structured Data Extraction</strong></summary>
+
+- 🤖 **LLM-Driven Extraction**: Supports all LLMs (open-source and proprietary) for structured data extraction.
+- 🧱 **Chunking Strategies**: Implements chunking (topic-based, regex, sentence-level) for targeted content processing.
+- 🌌 **Cosine Similarity**: Find relevant content chunks based on user queries for semantic extraction.
+- 🔎 **CSS-Based Extraction**: Fast schema-based data extraction using XPath and CSS selectors.
+- 🔧 **Schema Definition**: Define custom schemas for extracting structured JSON from repetitive patterns.
+
+</details>
+
+<details>
+<summary>🌐 <strong>Browser Integration</strong></summary>
+
+- 🖥️ **Managed Browser**: Use user-owned browsers with full control, avoiding bot detection.
+- 🔄 **Remote Browser Control**: Connect to Chrome Developer Tools Protocol for remote, large-scale data extraction.
+- 🔒 **Session Management**: Preserve browser states and reuse them for multi-step crawling.
+- 🧩 **Proxy Support**: Seamlessly connect to proxies with authentication for secure access.
+- ⚙️ **Full Browser Control**: Modify headers, cookies, user agents, and more for tailored crawling setups.
+- 🌍 **Multi-Browser Support**: Compatible with Chromium, Firefox, and WebKit.
+- 📐 **Dynamic Viewport Adjustment**: Automatically adjusts the browser viewport to match page content, ensuring complete rendering and capturing of all elements.
+
+</details>
+
+<details>
+<summary>🔎 <strong>Crawling & Scraping</strong></summary>
+
+- 🖼️ **Media Support**: Extract images, audio, videos, and responsive image formats like `srcset` and `picture`.
+- 🚀 **Dynamic Crawling**: Execute JS and wait for async or sync for dynamic content extraction.
+- 📸 **Screenshots**: Capture page screenshots during crawling for debugging or analysis.
+- 📂 **Raw Data Crawling**: Directly process raw HTML (`raw:`) or local files (`file://`).
+- 🔗 **Comprehensive Link Extraction**: Extracts internal, external links, and embedded iframe content.
+- 🛠️ **Customizable Hooks**: Define hooks at every step to customize crawling behavior.
+- 💾 **Caching**: Cache data for improved speed and to avoid redundant fetches.
+- 📄 **Metadata Extraction**: Retrieve structured metadata from web pages.
+- 📡 **IFrame Content Extraction**: Seamless extraction from embedded iframe content.
+- 🕵️ **Lazy Load Handling**: Waits for images to fully load, ensuring no content is missed due to lazy loading.
+- 🔄 **Full-Page Scanning**: Simulates scrolling to load and capture all dynamic content, perfect for infinite scroll pages.
+
+</details>
+
+<details>
+<summary>🚀 <strong>Deployment</strong></summary>
+
+- 🐳 **Dockerized Setup**: Optimized Docker image with API server for easy deployment.
+- 🔄 **API Gateway**: One-click deployment with secure token authentication for API-based workflows.
+- 🌐 **Scalable Architecture**: Designed for mass-scale production and optimized server performance.
+- ⚙️ **DigitalOcean Deployment**: Ready-to-deploy configurations for DigitalOcean and similar platforms.
+
+</details>
+
+<details>
+<summary>🎯 <strong>Additional Features</strong></summary>
+
+- 🕶️ **Stealth Mode**: Avoid bot detection by mimicking real users.
+- 🏷️ **Tag-Based Content Extraction**: Refine crawling based on custom tags, headers, or metadata.
+- 🔗 **Link Analysis**: Extract and analyze all links for detailed data exploration.
+- 🛡️ **Error Handling**: Robust error management for seamless execution.
+- 🔐 **CORS & Static Serving**: Supports filesystem-based caching and cross-origin requests.
+- 📖 **Clear Documentation**: Simplified and updated guides for onboarding and advanced usage.
+- 🙌 **Community Recognition**: Acknowledges contributors and pull requests for transparency.
+
+</details>
+
+## Try it Now!
+
+✨ Play around with this [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1SgRPrByQLzjRfwoRNq1wSGE9nYY_EE8C?usp=sharing)
+
+✨ Visit our [Documentation Website](https://crawl4ai.com/mkdocs/)
+
+## Installation 🛠️
+
+Crawl4AI offers flexible installation options to suit various use cases. You can install it as a Python package or use Docker.
+
+<details>
+<summary>🐍 <strong>Using pip</strong></summary>
+
+Choose the installation option that best fits your needs:
+
+### Basic Installation
+
+For basic web crawling and scraping tasks:
+
+```bash
+pip install crawl4ai
+crawl4ai-setup # Setup the browser
+```
+
+By default, this will install the asynchronous version of Crawl4AI, using Playwright for web crawling.
+
+👉 **Note**: When you install Crawl4AI, the `crawl4ai-setup` should automatically install and set up Playwright. However, if you encounter any Playwright-related errors, you can manually install it using one of these methods:
+
+1. Through the command line:
+
+   ```bash
+   playwright install
+   ```
+
+2. If the above doesn't work, try this more specific command:
+
+   ```bash
+   python -m playwright install chromium
+   ```
+
+This second method has proven to be more reliable in some cases.
+
+---
+
+### Installation with Synchronous Version
+
+The sync version is deprecated and will be removed in future versions. If you need the synchronous version using Selenium:
+
+```bash
+pip install crawl4ai[sync]
+```
+
+---
+
+### Development Installation
+
+For contributors who plan to modify the source code:
+
+```bash
+git clone https://github.com/unclecode/crawl4ai.git
+cd crawl4ai
+pip install -e .                    # Basic installation in editable mode
+```
+
+Install optional features:
+
+```bash
+pip install -e ".[torch]"           # With PyTorch features
+pip install -e ".[transformer]"     # With Transformer features
+pip install -e ".[cosine]"          # With cosine similarity features
+pip install -e ".[sync]"            # With synchronous crawling (Selenium)
+pip install -e ".[all]"             # Install all optional features
+```
+
+</details>
+
+<details>
+<summary>🐳 <strong>Docker Deployment</strong></summary>
+
+> 🚀 **Major Changes Coming!** We're developing a completely new Docker implementation that will make deployment even more efficient and seamless. The current Docker setup is being deprecated in favor of this new solution.
+
+### Current Docker Support
+
+The existing Docker implementation is being deprecated and will be replaced soon. If you still need to use Docker with the current version:
+
+- 📚 [Deprecated Docker Setup](./docs/deprecated/docker-deployment.md) - Instructions for the current Docker implementation
+- ⚠️ Note: This setup will be replaced in the next major release
+
+### What's Coming Next?
+
+Our new Docker implementation will bring:
+- Improved performance and resource efficiency
+- Streamlined deployment process
+- Better integration with Crawl4AI features
+- Enhanced scalability options
+
+Stay connected with our [GitHub repository](https://github.com/unclecode/crawl4ai) for updates!
+
+</details>
+
+---
+
+### Quick Test
+
+Run a quick test (works for both Docker options):
+
+```python
+import requests
+
+# Submit a crawl job
+response = requests.post(
+    "http://localhost:11235/crawl",
+    json={"urls": "https://example.com", "priority": 10}
+)
+task_id = response.json()["task_id"]
+
+# Continue polling until the task is complete (status="completed")
+result = requests.get(f"http://localhost:11235/task/{task_id}")
+```
+
+For more examples, see our [Docker Examples](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/docker_example.py). For advanced configuration, environment variables, and usage examples, see our [Docker Deployment Guide](https://crawl4ai.com/mkdocs/basic/docker-deployment/).
+
+</details>
+
+
+## 🔬 Advanced Usage Examples 🔬
+
+You can check the project structure in the directory [https://github.com/unclecode/crawl4ai/docs/examples](docs/examples). Over there, you can find a variety of examples; here, some popular examples are shared.
+
+<details>
+<summary>📝 <strong>Heuristic Markdown Generation with Clean and Fit Markdown</strong></summary>
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+from crawl4ai.content_filter_strategy import PruningContentFilter, BM25ContentFilter
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+async def main():
+    browser_config = BrowserConfig(
+        headless=True,  
+        verbose=True,
+    )
+    run_config = CrawlerRunConfig(
+        cache_mode=CacheMode.ENABLED,
+        markdown_generator=DefaultMarkdownGenerator(
+            content_filter=PruningContentFilter(threshold=0.48, threshold_type="fixed", min_word_threshold=0)
+        ),
+        # markdown_generator=DefaultMarkdownGenerator(
+        #     content_filter=BM25ContentFilter(user_query="WHEN_WE_FOCUS_BASED_ON_A_USER_QUERY", bm25_threshold=1.0)
+        # ),
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://docs.micronaut.io/4.7.6/guide/",
+            config=run_config
+        )
+        print(len(result.markdown))
+        print(len(result.fit_markdown))
+        print(len(result.markdown_v2.fit_markdown))
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+</details>
+
+<details>
+<summary>🖥️ <strong>Executing JavaScript & Extract Structured Data without LLMs</strong></summary>
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+import json
+
+async def main():
+    schema = {
+    "name": "KidoCode Courses",
+    "baseSelector": "section.charge-methodology .w-tab-content > div",
+    "fields": [
+        {
+            "name": "section_title",
+            "selector": "h3.heading-50",
+            "type": "text",
+        },
+        {
+            "name": "section_description",
+            "selector": ".charge-content",
+            "type": "text",
+        },
+        {
+            "name": "course_name",
+            "selector": ".text-block-93",
+            "type": "text",
+        },
+        {
+            "name": "course_description",
+            "selector": ".course-content-text",
+            "type": "text",
+        },
+        {
+            "name": "course_icon",
+            "selector": ".image-92",
+            "type": "attribute",
+            "attribute": "src"
+        }
+    }
+}
+
+    extraction_strategy = JsonCssExtractionStrategy(schema, verbose=True)
+
+    browser_config = BrowserConfig(
+        headless=False,
+        verbose=True
+    )
+    run_config = CrawlerRunConfig(
+        extraction_strategy=extraction_strategy,
+        js_code=["""(async () => {const tabs = document.querySelectorAll("section.charge-methodology .tabs-menu-3 > div");for(let tab of tabs) {tab.scrollIntoView();tab.click();await new Promise(r => setTimeout(r, 500));}})();"""],
+        cache_mode=CacheMode.BYPASS
+    )
+        
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        
+        result = await crawler.arun(
+            url="https://www.kidocode.com/degrees/technology",
+            config=run_config
+        )
+
+        companies = json.loads(result.extracted_content)
+        print(f"Successfully extracted {len(companies)} companies")
+        print(json.dumps(companies[0], indent=2))
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+</details>
+
+<details>
+<summary>📚 <strong>Extracting Structured Data with LLMs</strong></summary>
+
+```python
+import os
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+from crawl4ai.extraction_strategy import LLMExtractionStrategy
+from pydantic import BaseModel, Field
+
+class OpenAIModelFee(BaseModel):
+    model_name: str = Field(..., description="Name of the OpenAI model.")
+    input_fee: str = Field(..., description="Fee for input token for the OpenAI model.")
+    output_fee: str = Field(..., description="Fee for output token for the OpenAI model.")
+
+async def main():
+    browser_config = BrowserConfig(verbose=True)
+    run_config = CrawlerRunConfig(
+        word_count_threshold=1,
+        extraction_strategy=LLMExtractionStrategy(
+            # Here you can use any provider that Litellm library supports, for instance: ollama/qwen2
+            # provider="ollama/qwen2", api_token="no-token", 
+            provider="openai/gpt-4o", api_token=os.getenv('OPENAI_API_KEY'), 
+            schema=OpenAIModelFee.schema(),
+            extraction_type="schema",
+            instruction="""From the crawled content, extract all mentioned model names along with their fees for input and output tokens. 
+            Do not miss any models in the entire content. One extracted model JSON format should look like this: 
+            {"model_name": "GPT-4", "input_fee": "US$10.00 / 1M tokens", "output_fee": "US$30.00 / 1M tokens"}."""
+        ),            
+        cache_mode=CacheMode.BYPASS,
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url='https://openai.com/api/pricing/',
+            config=run_config
+        )
+        print(result.extracted_content)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+</details>
+
+<details>
+<summary>🤖 <strong>Using You own Browswer with Custome User Profile</strong></summary>
+
+```python
+import os, sys
+from pathlib import Path
+import asyncio, time
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+
+async def test_news_crawl():
+    # Create a persistent user data directory
+    user_data_dir = os.path.join(Path.home(), ".crawl4ai", "browser_profile")
+    os.makedirs(user_data_dir, exist_ok=True)
+
+    browser_config = BrowserConfig(
+        verbose=True,
+        headless=True,
+        user_data_dir=user_data_dir,
+        use_persistent_context=True,
+    )
+    run_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        url = "ADDRESS_OF_A_CHALLENGING_WEBSITE"
+        
+        result = await crawler.arun(
+            url,
+            config=run_config,
+            magic=True,
+        )
+        
+        print(f"Successfully crawled {url}")
+        print(f"Content length: {len(result.markdown)}")
+```
+
+</details>
+
+
+## ✨ Recent Updates   
+
+- 🔒 **Enhanced SSL & Security**: New SSL certificate handling with custom paths and validation options for secure crawling
+- 🔍 **Smart Content Filtering**: Advanced filtering system with regex support and efficient chunking strategies
+- 📦 **Improved JSON Extraction**: Support for complex JSONPath, JSON-CSS, and Microdata extraction
+- 🏗️ **New Field Types**: Added `computed`, `conditional`, `aggregate`, and `template` field types
+- ⚡ **Performance Boost**: Optimized caching, parallel processing, and memory management
+- 🐛 **Better Error Handling**: Enhanced debugging capabilities with detailed error tracking
+- 🔐 **Security Features**: Improved input validation and safe expression evaluation
+
+Read the full details of this release in our [0.4.24 Release Notes](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md).
+
+## 📖 Documentation & Roadmap 
+
+> 🚨 **Documentation Update Alert**: We're undertaking a major documentation overhaul next week to reflect recent updates and improvements. Stay tuned for a more comprehensive and up-to-date guide!
+
+For current documentation, including installation instructions, advanced features, and API reference, visit our [Documentation Website](https://crawl4ai.com/mkdocs/).
+
+To check our development plans and upcoming features, visit our [Roadmap](https://github.com/unclecode/crawl4ai/blob/main/ROADMAP.md).
+
+<details>
+<summary>📈 <strong>Development TODOs</strong></summary>
+
+- [x] 0. Graph Crawler: Smart website traversal using graph search algorithms for comprehensive nested page extraction
+- [ ] 1. Question-Based Crawler: Natural language driven web discovery and content extraction
+- [ ] 2. Knowledge-Optimal Crawler: Smart crawling that maximizes knowledge while minimizing data extraction
+- [ ] 3. Agentic Crawler: Autonomous system for complex multi-step crawling operations
+- [ ] 4. Automated Schema Generator: Convert natural language to extraction schemas
+- [ ] 5. Domain-Specific Scrapers: Pre-configured extractors for common platforms (academic, e-commerce)
+- [ ] 6. Web Embedding Index: Semantic search infrastructure for crawled content
+- [ ] 7. Interactive Playground: Web UI for testing, comparing strategies with AI assistance
+- [ ] 8. Performance Monitor: Real-time insights into crawler operations
+- [ ] 9. Cloud Integration: One-click deployment solutions across cloud providers
+- [ ] 10. Sponsorship Program: Structured support system with tiered benefits
+- [ ] 11. Educational Content: "How to Crawl" video series and interactive tutorials
+
+</details>
+
+## 🤝 Contributing 
+
+We welcome contributions from the open-source community. Check out our [contribution guidelines](https://github.com/unclecode/crawl4ai/blob/main/CONTRIBUTING.md) for more information.
+
+## 📄 License 
+
+Crawl4AI is released under the [Apache 2.0 License](https://github.com/unclecode/crawl4ai/blob/main/LICENSE).
+
+## 📧 Contact 
+
+For questions, suggestions, or feedback, feel free to reach out:
+
+- GitHub: [unclecode](https://github.com/unclecode)
+- Twitter: [@unclecode](https://twitter.com/unclecode)
+- Website: [crawl4ai.com](https://crawl4ai.com)
+
+Happy Crawling! 🕸️🚀
+
+## 🗾 Mission
+
+Our mission is to unlock the value of personal and enterprise data by transforming digital footprints into structured, tradeable assets. Crawl4AI empowers individuals and organizations with open-source tools to extract and structure data, fostering a shared data economy.  
+
+We envision a future where AI is powered by real human knowledge, ensuring data creators directly benefit from their contributions. By democratizing data and enabling ethical sharing, we are laying the foundation for authentic AI advancement.
+
+<details>
+<summary>🔑 <strong>Key Opportunities</strong></summary>
+ 
+- **Data Capitalization**: Transform digital footprints into measurable, valuable assets.  
+- **Authentic AI Data**: Provide AI systems with real human insights.  
+- **Shared Economy**: Create a fair data marketplace that benefits data creators.  
+
+</details>
+
+<details>
+<summary>🚀 <strong>Development Pathway</strong></summary>
+
+1. **Open-Source Tools**: Community-driven platforms for transparent data extraction.  
+2. **Digital Asset Structuring**: Tools to organize and value digital knowledge.  
+3. **Ethical Data Marketplace**: A secure, fair platform for exchanging structured data.  
+
+For more details, see our [full mission statement](./MISSION.md).
+</details>
+
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=unclecode/crawl4ai&type=Date)](https://star-history.com/#unclecode/crawl4ai&Date)

ROADMAP.md

@@ -0,0 +1,503 @@
+# Crawl4AI Strategic Roadmap
+
+```mermaid
+%%{init: {'themeVariables': { 'fontSize': '14px'}}}%%
+graph TD
+    subgraph A1[Advanced Crawling Systems 🔧]
+        A["`
+        • Graph Crawler ✓
+        • Question-Based Crawler
+        • Knowledge-Optimal Crawler
+        • Agentic Crawler
+        `"]
+    end
+
+    subgraph A2[Specialized Features 🛠️]
+        B["`
+        • Automated Schema Generator
+        • Domain-Specific Scrapers
+        • 
+        • 
+        `"]
+    end
+
+    subgraph A3[Development Tools 🔨]
+        C["`
+        • Interactive Playground
+        • Performance Monitor
+        • Cloud Integration
+        • 
+        `"]
+    end
+
+    subgraph A4[Community & Growth 🌱]
+        D["`
+        • Sponsorship Program
+        • Educational Content
+        • 
+        • 
+        `"]
+    end
+
+    classDef default fill:#f9f9f9,stroke:#333,stroke-width:2px
+    classDef section fill:#f0f0f0,stroke:#333,stroke-width:4px,rx:10
+    class A1,A2,A3,A4 section
+
+    %% Layout hints
+    A1 --> A2[" "]
+    A3 --> A4[" "]
+    linkStyle 0,1 stroke:none
+```
+
+Crawl4AI is evolving to provide more intelligent, efficient, and versatile web crawling capabilities. This roadmap outlines the key developments and features planned for the project, organized into strategic sections that build upon our current foundation.
+
+## 1. Advanced Crawling Systems 🔧
+
+This section introduces three powerful crawling systems that extend Crawl4AI's capabilities from basic web crawling to intelligent, purpose-driven data extraction.
+
+### 1.1 Question-Based Crawler
+The Question-Based Crawler enhances our core engine by enabling automatic discovery and extraction of relevant web content based on natural language questions.
+
+Key Features:
+- SerpiAPI integration for intelligent web search
+- Relevancy scoring for search results
+- Automatic URL discovery and prioritization
+- Cross-source validation
+
+```python
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.discovery import QuestionBasedDiscovery
+
+async with AsyncWebCrawler() as crawler:
+    discovery = QuestionBasedDiscovery(crawler)
+    results = await discovery.arun(
+        question="What are the system requirements for major cloud providers' GPU instances?",
+        max_urls=5,
+        relevance_threshold=0.7
+    )
+    
+    for result in results:
+        print(f"Source: {result.url} (Relevance: {result.relevance_score})")
+        print(f"Content: {result.markdown}\n")
+```
+
+### 1.2 Knowledge-Optimal Crawler
+An intelligent crawling system that solves the optimization problem of minimizing data extraction while maximizing knowledge acquisition for specific objectives.
+
+Key Features:
+- Smart content prioritization
+- Minimal data extraction for maximum knowledge
+- Probabilistic relevance assessment
+- Objective-driven crawling paths
+
+```python
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.optimization import KnowledgeOptimizer
+
+async with AsyncWebCrawler() as crawler:
+    optimizer = KnowledgeOptimizer(
+        objective="Understand GPU instance pricing and limitations across cloud providers",
+        required_knowledge=[
+            "pricing structure",
+            "GPU specifications",
+            "usage limits",
+            "availability zones"
+        ],
+        confidence_threshold=0.85
+    )
+    
+    result = await crawler.arun(
+        urls=[
+            "https://aws.amazon.com/ec2/pricing/",
+            "https://cloud.google.com/gpu",
+            "https://azure.microsoft.com/pricing/"
+        ],
+        optimizer=optimizer,
+        optimization_mode="minimal_extraction"
+    )
+    
+    print(f"Knowledge Coverage: {result.knowledge_coverage}")
+    print(f"Data Efficiency: {result.efficiency_ratio}")
+    print(f"Extracted Content: {result.optimal_content}")
+```
+
+### 1.3 Agentic Crawler
+An autonomous system capable of understanding complex goals and automatically planning and executing multi-step crawling operations.
+
+Key Features:
+- Autonomous goal interpretation
+- Dynamic step planning
+- Interactive navigation capabilities
+- Visual recognition and interaction
+- Automatic error recovery
+
+```python
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.agents import CrawlerAgent
+
+async with AsyncWebCrawler() as crawler:
+    agent = CrawlerAgent(crawler)
+    
+    # Automatic planning and execution
+    result = await agent.arun(
+        goal="Find research papers about quantum computing published in 2023 with more than 50 citations",
+        auto_retry=True
+    )
+    print("Generated Plan:", result.executed_steps)
+    print("Extracted Data:", result.data)
+    
+    # Using custom steps with automatic execution
+    result = await agent.arun(
+        goal="Extract conference deadlines from ML conferences",
+        custom_plan=[
+            "Navigate to conference page",
+            "Find important dates section",
+            "Extract submission deadlines",
+            "Verify dates are for 2024"
+        ]
+    )
+    
+    # Monitoring execution
+    print("Step Completion:", result.step_status)
+    print("Execution Time:", result.execution_time)
+    print("Success Rate:", result.success_rate)
+```
+
+# Section 2: Specialized Features 🛠️
+
+This section introduces specialized tools and features that enhance Crawl4AI's capabilities for specific use cases and data extraction needs.
+
+### 2.1 Automated Schema Generator
+A system that automatically generates JsonCssExtractionStrategy schemas from natural language descriptions, making structured data extraction accessible to all users.
+
+Key Features:
+- Natural language schema generation
+- Automatic pattern detection
+- Predefined schema templates
+- Chrome extension for visual schema building
+
+```python
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.schema import SchemaGenerator
+
+# Generate schema from natural language description
+generator = SchemaGenerator()
+schema = await generator.generate(
+    url="https://news-website.com",
+    description="For each news article on the page, I need the headline, publication date, and main image"
+)
+
+# Use generated schema with crawler
+async with AsyncWebCrawler() as crawler:
+    result = await crawler.arun(
+        url="https://news-website.com",
+        extraction_strategy=schema
+    )
+
+# Example of generated schema:
+"""
+{
+    "name": "News Article Extractor",
+    "baseSelector": "article.news-item",
+    "fields": [
+        {
+            "name": "headline",
+            "selector": "h2.article-title",
+            "type": "text"
+        },
+        {
+            "name": "date",
+            "selector": "span.publish-date",
+            "type": "text"
+        },
+        {
+            "name": "image",
+            "selector": "img.article-image",
+            "type": "attribute",
+            "attribute": "src"
+        }
+    ]
+}
+"""
+```
+
+### 2.2 Domain Specific Scrapers
+Specialized extraction strategies optimized for common website types and platforms, providing consistent and reliable data extraction without additional configuration.
+
+Key Features:
+- Pre-configured extractors for popular platforms
+- Academic site specialization (arXiv, NCBI)
+- E-commerce standardization
+- Documentation site handling
+
+```python
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.extractors import AcademicExtractor, EcommerceExtractor
+
+async with AsyncWebCrawler() as crawler:
+    # Academic paper extraction
+    papers = await crawler.arun(
+        url="https://arxiv.org/list/cs.AI/recent",
+        extractor="academic",  # Built-in extractor type
+        site_type="arxiv",     # Specific site optimization
+        extract_fields=[
+            "title", 
+            "authors", 
+            "abstract", 
+            "citations"
+        ]
+    )
+    
+    # E-commerce product data
+    products = await crawler.arun(
+        url="https://store.example.com/products",
+        extractor="ecommerce",
+        extract_fields=[
+            "name",
+            "price",
+            "availability",
+            "reviews"
+        ]
+    )
+```
+
+### 2.3 Web Embedding Index
+Creates and maintains a semantic search infrastructure for crawled content, enabling efficient retrieval and querying of web content through vector embeddings.
+
+Key Features:
+- Automatic embedding generation
+- Intelligent content chunking
+- Efficient vector storage and indexing
+- Semantic search capabilities
+
+```python
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.indexing import WebIndex
+
+# Initialize and build index
+index = WebIndex(model="efficient-mini")
+
+async with AsyncWebCrawler() as crawler:
+    # Crawl and index content
+    await index.build(
+        urls=["https://docs.example.com"],
+        crawler=crawler,
+        options={
+            "chunk_method": "semantic",
+            "update_policy": "incremental",
+            "embedding_batch_size": 100
+        }
+    )
+
+    # Search through indexed content
+    results = await index.search(
+        query="How to implement OAuth authentication?",
+        filters={
+            "content_type": "technical",
+            "recency": "6months"
+        },
+        top_k=5
+    )
+
+    # Get similar content
+    similar = await index.find_similar(
+        url="https://docs.example.com/auth/oauth",
+        threshold=0.85
+    )
+```
+
+Each of these specialized features builds upon Crawl4AI's core functionality while providing targeted solutions for specific use cases. They can be used independently or combined for more complex data extraction and processing needs.
+
+# Section 3: Development Tools 🔧
+
+This section covers tools designed to enhance the development experience, monitoring, and deployment of Crawl4AI applications.
+
+### 3.1 Crawl4AI Playground 🎮
+
+The Crawl4AI Playground is an interactive web-based development environment that simplifies web scraping experimentation, development, and deployment. With its intuitive interface and AI-powered assistance, users can quickly prototype, test, and deploy web scraping solutions.
+
+#### Key Features 🌟
+
+##### Visual Strategy Builder
+- Interactive point-and-click interface for building extraction strategies
+- Real-time preview of selected elements
+- Side-by-side comparison of different extraction approaches
+- Visual validation of CSS selectors and XPath queries
+
+##### AI Assistant Integration
+- Strategy recommendations based on target website analysis
+- Parameter optimization suggestions
+- Best practices guidance for specific use cases
+- Automated error detection and resolution
+- Performance optimization tips
+
+##### Real-Time Testing & Validation
+- Live preview of extraction results
+- Side-by-side comparison of multiple strategies
+- Performance metrics visualization
+- Automatic validation of extracted data
+- Error detection and debugging tools
+
+##### Project Management
+- Save and organize multiple scraping projects
+- Version control for configurations
+- Export/import project settings
+- Share configurations with team members
+- Project templates for common use cases
+
+##### Deployment Pipeline
+- One-click deployment to various environments
+- Docker container generation
+- Cloud deployment templates (AWS, GCP, Azure)
+- Scaling configuration management
+- Monitoring setup automation
+
+
+### 3.2 Performance Monitoring System
+A comprehensive monitoring solution providing real-time insights into crawler operations, resource usage, and system health through both CLI and GUI interfaces.
+
+Key Features:
+- Real-time resource tracking
+- Active crawl monitoring
+- Performance statistics
+- Customizable alerting system
+
+```python
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.monitor import CrawlMonitor
+
+# Initialize monitoring
+monitor = CrawlMonitor()
+
+# Start monitoring with CLI interface
+await monitor.start(
+    mode="cli",  # or "gui"
+    refresh_rate="1s",
+    metrics={
+        "resources": ["cpu", "memory", "network"],
+        "crawls": ["active", "queued", "completed"],
+        "performance": ["success_rate", "response_times"]
+    }
+)
+
+# Example CLI output:
+"""
+Crawl4AI Monitor (Live) - Press Q to exit
+────────────────────────────────────────
+System Usage:
+ ├─ CPU: ███████░░░ 70%
+ └─ Memory: ████░░░░░ 2.1GB/8GB
+
+Active Crawls:
+ID    URL                   Status    Progress
+001   docs.example.com     🟢 Active   75%
+002   api.service.com      🟡 Queue    -
+
+Metrics (Last 5min):
+ ├─ Success Rate: 98%
+ ├─ Avg Response: 0.6s
+ └─ Pages/sec: 8.5
+"""
+```
+
+### 3.3 Cloud Integration
+Streamlined deployment tools for setting up Crawl4AI in various cloud environments, with support for scaling and monitoring.
+
+Key Features:
+- One-click deployment solutions
+- Auto-scaling configuration
+- Load balancing setup
+- Cloud-specific optimizations
+- Monitoring integration
+
+```python
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.deploy import CloudDeployer
+
+# Initialize deployer
+deployer = CloudDeployer()
+
+# Deploy crawler service
+deployment = await deployer.deploy(
+    service_name="crawler-cluster",
+    platform="aws",  # or "gcp", "azure"
+    config={
+        "instance_type": "compute-optimized",
+        "auto_scaling": {
+            "min_instances": 2,
+            "max_instances": 10,
+            "scale_based_on": "cpu_usage"
+        },
+        "region": "us-east-1",
+        "monitoring": True
+    }
+)
+
+# Get deployment status and endpoints
+print(f"Service Status: {deployment.status}")
+print(f"API Endpoint: {deployment.endpoint}")
+print(f"Monitor URL: {deployment.monitor_url}")
+```
+
+These development tools work together to provide a comprehensive environment for developing, testing, monitoring, and deploying Crawl4AI applications. The Playground helps users experiment and generate optimal configurations, the Performance Monitor ensures smooth operation, and the Cloud Integration tools simplify deployment and scaling.
+
+# Section 4: Community & Growth 🌱
+
+This section outlines initiatives designed to build and support the Crawl4AI community, provide educational resources, and ensure sustainable project growth.
+
+### 4.1 Sponsorship Program
+A structured program to support ongoing development and maintenance of Crawl4AI while providing valuable benefits to sponsors.
+
+Key Features:
+- Multiple sponsorship tiers
+- Sponsor recognition system
+- Priority support for sponsors
+- Early access to new features
+- Custom feature development opportunities
+
+Program Structure (not yet finalized):
+```
+Sponsorship Tiers:
+
+🥉 Bronze Supporter
+- GitHub Sponsor badge
+- Priority issue response
+- Community Discord role
+
+🥈 Silver Supporter
+- All Bronze benefits
+- Technical support channel
+- Vote on roadmap priorities
+- Early access to beta features
+
+🥇 Gold Supporter
+- All Silver benefits
+- Custom feature requests
+- Direct developer access
+- Private support sessions
+
+💎 Diamond Partner
+- All Gold benefits
+- Custom development
+- On-demand consulting
+- Integration support
+```
+
+### 4.2 "How to Crawl" Video Series
+A comprehensive educational resource teaching users how to effectively use Crawl4AI for various web scraping and data extraction scenarios.
+
+Key Features:
+- Step-by-step tutorials
+- Real-world use cases
+- Best practices
+- Integration guides
+- Advanced feature deep-dives
+
+These community initiatives are designed to:
+- Provide comprehensive learning resources
+- Foster a supportive user community
+- Ensure sustainable project development
+- Share knowledge and best practices
+- Create opportunities for collaboration
+
+The combination of structured support through sponsorship, educational content through video series, and interactive learning through the playground creates a robust ecosystem for both new and experienced users of Crawl4AI.

crawl4ai/__init__.py

@@ -0,0 +1,46 @@
+# __init__.py
+
+from .async_webcrawler import AsyncWebCrawler, CacheMode
+from .async_configs import BrowserConfig, CrawlerRunConfig
+from .extraction_strategy import ExtractionStrategy, LLMExtractionStrategy, CosineStrategy, JsonCssExtractionStrategy
+from .chunking_strategy import ChunkingStrategy, RegexChunking
+from .markdown_generation_strategy import DefaultMarkdownGenerator
+from .content_filter_strategy import PruningContentFilter, BM25ContentFilter
+from .models import CrawlResult
+from .__version__ import __version__
+
+__all__ = [
+    "AsyncWebCrawler",
+    "CrawlResult",
+    "CacheMode",
+    'BrowserConfig',
+    'CrawlerRunConfig',
+    'ExtractionStrategy',
+    'LLMExtractionStrategy',
+    'CosineStrategy',
+    'JsonCssExtractionStrategy',
+    'ChunkingStrategy',
+    'RegexChunking',
+    'DefaultMarkdownGenerator',
+    'PruningContentFilter',
+    'BM25ContentFilter',
+]
+
+def is_sync_version_installed():
+    try:
+        import selenium
+        return True
+    except ImportError:
+        return False
+
+if is_sync_version_installed():
+    try:
+        from .web_crawler import WebCrawler
+        __all__.append("WebCrawler")
+    except ImportError:
+        import warnings
+        print("Warning: Failed to import WebCrawler even though selenium is installed. This might be due to other missing dependencies.")
+else:
+    WebCrawler = None
+    # import warnings
+    # print("Warning: Synchronous WebCrawler is not available. Install crawl4ai[sync] for synchronous support. However, please note that the synchronous version will be deprecated soon.")

crawl4ai/__version__.py

@@ -0,0 +1,2 @@
+# crawl4ai/_version.py
+__version__ = "0.4.247"

crawl4ai/async_configs.py

@@ -0,0 +1,603 @@
+from .config import (
+    MIN_WORD_THRESHOLD,
+    IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD,
+    SCREENSHOT_HEIGHT_TRESHOLD,
+    PAGE_TIMEOUT,
+    IMAGE_SCORE_THRESHOLD,
+    SOCIAL_MEDIA_DOMAINS,
+
+)
+from .user_agent_generator import UserAgentGenerator
+from .extraction_strategy import ExtractionStrategy
+from .chunking_strategy import ChunkingStrategy
+from .markdown_generation_strategy import MarkdownGenerationStrategy
+from typing import Union, List
+
+
+class BrowserConfig:
+    """
+    Configuration class for setting up a browser instance and its context in AsyncPlaywrightCrawlerStrategy.
+
+    This class centralizes all parameters that affect browser and context creation. Instead of passing
+    scattered keyword arguments, users can instantiate and modify this configuration object. The crawler
+    code will then reference these settings to initialize the browser in a consistent, documented manner.
+
+    Attributes:
+        browser_type (str): The type of browser to launch. Supported values: "chromium", "firefox", "webkit".
+                            Default: "chromium".
+        headless (bool): Whether to run the browser in headless mode (no visible GUI).
+                         Default: True.
+        use_managed_browser (bool): Launch the browser using a managed approach (e.g., via CDP), allowing
+                                    advanced manipulation. Default: False.
+        debugging_port (int): Port for the browser debugging protocol. Default: 9222.
+        use_persistent_context (bool): Use a persistent browser context (like a persistent profile).
+                                       Automatically sets use_managed_browser=True. Default: False.
+        user_data_dir (str or None): Path to a user data directory for persistent sessions. If None, a
+                                     temporary directory may be used. Default: None.
+        chrome_channel (str): The Chrome channel to launch (e.g., "chrome", "msedge"). Only applies if browser_type
+                              is "chromium". Default: "chromium".
+        channel (str): The channel to launch (e.g., "chromium", "chrome", "msedge"). Only applies if browser_type
+                              is "chromium". Default: "chromium".
+        proxy (str or None): Proxy server URL (e.g., "http://username:password@proxy:port"). If None, no proxy is used.
+                             Default: None.
+        proxy_config (dict or None): Detailed proxy configuration, e.g. {"server": "...", "username": "..."}.
+                                     If None, no additional proxy config. Default: None.
+        viewport_width (int): Default viewport width for pages. Default: 1080.
+        viewport_height (int): Default viewport height for pages. Default: 600.
+        verbose (bool): Enable verbose logging.
+                        Default: True.
+        accept_downloads (bool): Whether to allow file downloads. If True, requires a downloads_path.
+                                 Default: False.
+        downloads_path (str or None): Directory to store downloaded files. If None and accept_downloads is True,
+                                      a default path will be created. Default: None.
+        storage_state (str or dict or None): Path or object describing storage state (cookies, localStorage).
+                                             Default: None.
+        ignore_https_errors (bool): Ignore HTTPS certificate errors. Default: True.
+        java_script_enabled (bool): Enable JavaScript execution in pages. Default: True.
+        cookies (list): List of cookies to add to the browser context. Each cookie is a dict with fields like
+                        {"name": "...", "value": "...", "url": "..."}.
+                        Default: [].
+        headers (dict): Extra HTTP headers to apply to all requests in this context.
+                        Default: {}.
+        user_agent (str): Custom User-Agent string to use. Default: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+                           "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36".
+        user_agent_mode (str or None): Mode for generating the user agent (e.g., "random"). If None, use the provided
+                                       user_agent as-is. Default: None.
+        user_agent_generator_config (dict or None): Configuration for user agent generation if user_agent_mode is set.
+                                                    Default: None.
+        text_mode (bool): If True, disables images and other rich content for potentially faster load times.
+                          Default: False.
+        light_mode (bool): Disables certain background features for performance gains. Default: False.
+        extra_args (list): Additional command-line arguments passed to the browser.
+                           Default: [].
+    """
+
+    def __init__(
+        self,
+        browser_type: str = "chromium",
+        headless: bool = True,
+        use_managed_browser: bool = False,
+        use_persistent_context: bool = False,
+        user_data_dir: str = None,
+        chrome_channel: str = "chromium",
+        channel: str = "chromium",
+        proxy: str = None,
+        proxy_config: dict = None,
+        viewport_width: int = 1080,
+        viewport_height: int = 600, 
+        accept_downloads: bool = False,
+        downloads_path: str = None,
+        storage_state=None,
+        ignore_https_errors: bool = True,
+        java_script_enabled: bool = True,
+        sleep_on_close: bool = False,
+        verbose: bool = True,
+        cookies: list = None,
+        headers: dict = None,
+        user_agent: str = (
+            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) AppleWebKit/537.36 "
+            "(KHTML, like Gecko) Chrome/116.0.5845.187 Safari/604.1 Edg/117.0.2045.47"
+        ),
+        user_agent_mode: str = None,
+        user_agent_generator_config: dict = None,
+        text_mode: bool = False,
+        light_mode: bool = False,
+        extra_args: list = None,
+        debugging_port : int = 9222,
+    ):
+        self.browser_type = browser_type
+        self.headless = headless
+        self.use_managed_browser = use_managed_browser
+        self.use_persistent_context = use_persistent_context
+        self.user_data_dir = user_data_dir
+        self.chrome_channel = chrome_channel or self.browser_type or "chromium"
+        self.channel = channel or self.browser_type or "chromium"
+        self.proxy = proxy
+        self.proxy_config = proxy_config
+        self.viewport_width = viewport_width
+        self.viewport_height = viewport_height
+        self.accept_downloads = accept_downloads
+        self.downloads_path = downloads_path
+        self.storage_state = storage_state
+        self.ignore_https_errors = ignore_https_errors
+        self.java_script_enabled = java_script_enabled
+        self.cookies = cookies if cookies is not None else []
+        self.headers = headers if headers is not None else {}
+        self.user_agent = user_agent
+        self.user_agent_mode = user_agent_mode
+        self.user_agent_generator_config = user_agent_generator_config
+        self.text_mode = text_mode
+        self.light_mode = light_mode
+        self.extra_args = extra_args if extra_args is not None else []
+        self.sleep_on_close = sleep_on_close
+        self.verbose = verbose
+        self.debugging_port = debugging_port
+
+        user_agenr_generator = UserAgentGenerator()
+        if self.user_agent_mode != "random" and self.user_agent_generator_config:
+            self.user_agent = user_agenr_generator.generate(
+                **(self.user_agent_generator_config or {})
+            )
+        elif self.user_agent_mode == "random":
+            self.user_agent = user_agenr_generator.generate()
+        else:
+            pass
+        
+        self.browser_hint = user_agenr_generator.generate_client_hints(self.user_agent)
+        self.headers.setdefault("sec-ch-ua", self.browser_hint)
+
+        # If persistent context is requested, ensure managed browser is enabled
+        if self.use_persistent_context:
+            self.use_managed_browser = True
+
+    @staticmethod
+    def from_kwargs(kwargs: dict) -> "BrowserConfig":
+        return BrowserConfig(
+            browser_type=kwargs.get("browser_type", "chromium"),
+            headless=kwargs.get("headless", True),
+            use_managed_browser=kwargs.get("use_managed_browser", False),
+            use_persistent_context=kwargs.get("use_persistent_context", False),
+            user_data_dir=kwargs.get("user_data_dir"),
+            chrome_channel=kwargs.get("chrome_channel", "chromium"),
+            channel=kwargs.get("channel", "chromium"),
+            proxy=kwargs.get("proxy"),
+            proxy_config=kwargs.get("proxy_config"),
+            viewport_width=kwargs.get("viewport_width", 1080),
+            viewport_height=kwargs.get("viewport_height", 600),
+            accept_downloads=kwargs.get("accept_downloads", False),
+            downloads_path=kwargs.get("downloads_path"),
+            storage_state=kwargs.get("storage_state"),
+            ignore_https_errors=kwargs.get("ignore_https_errors", True),
+            java_script_enabled=kwargs.get("java_script_enabled", True),
+            cookies=kwargs.get("cookies", []),
+            headers=kwargs.get("headers", {}),
+            user_agent=kwargs.get(
+                "user_agent",
+                "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+                "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
+            ),
+            user_agent_mode=kwargs.get("user_agent_mode"),
+            user_agent_generator_config=kwargs.get("user_agent_generator_config"),
+            text_mode=kwargs.get("text_mode", False),
+            light_mode=kwargs.get("light_mode", False),
+            extra_args=kwargs.get("extra_args", []),
+        )
+
+
+class CrawlerRunConfig:
+    """
+    Configuration class for controlling how the crawler runs each crawl operation.
+    This includes parameters for content extraction, page manipulation, waiting conditions,
+    caching, and other runtime behaviors.
+
+    This centralizes parameters that were previously scattered as kwargs to `arun()` and related methods.
+    By using this class, you have a single place to understand and adjust the crawling options.
+
+    Attributes:
+        # Content Processing Parameters
+        word_count_threshold (int): Minimum word count threshold before processing content.
+                                    Default: MIN_WORD_THRESHOLD (typically 200).
+        extraction_strategy (ExtractionStrategy or None): Strategy to extract structured data from crawled pages.
+                                                          Default: None (NoExtractionStrategy is used if None).
+        chunking_strategy (ChunkingStrategy): Strategy to chunk content before extraction.
+                                              Default: RegexChunking().
+        markdown_generator (MarkdownGenerationStrategy): Strategy for generating markdown.
+                                                         Default: None.
+        content_filter (RelevantContentFilter or None): Optional filter to prune irrelevant content.
+                                                        Default: None.
+        only_text (bool): If True, attempt to extract text-only content where applicable.
+                          Default: False.
+        css_selector (str or None): CSS selector to extract a specific portion of the page.
+                                    Default: None.
+        excluded_tags (list of str or None): List of HTML tags to exclude from processing.
+                                             Default: None.
+        excluded_selector (str or None): CSS selector to exclude from processing.
+                                         Default: None.
+        keep_data_attributes (bool): If True, retain `data-*` attributes while removing unwanted attributes.
+                                     Default: False.
+        remove_forms (bool): If True, remove all `<form>` elements from the HTML.
+                             Default: False.
+        prettiify (bool): If True, apply `fast_format_html` to produce prettified HTML output.
+                          Default: False.
+        parser_type (str): Type of parser to use for HTML parsing.
+                           Default: "lxml".
+
+        # Caching Parameters
+        cache_mode (CacheMode or None): Defines how caching is handled.
+                                        If None, defaults to CacheMode.ENABLED internally.
+                                        Default: None.
+        session_id (str or None): Optional session ID to persist the browser context and the created
+                                  page instance. If the ID already exists, the crawler does not
+                                  create a new page and uses the current page to preserve the state.
+        bypass_cache (bool): Legacy parameter, if True acts like CacheMode.BYPASS.
+                             Default: False.
+        disable_cache (bool): Legacy parameter, if True acts like CacheMode.DISABLED.
+                              Default: False.
+        no_cache_read (bool): Legacy parameter, if True acts like CacheMode.WRITE_ONLY.
+                              Default: False.
+        no_cache_write (bool): Legacy parameter, if True acts like CacheMode.READ_ONLY.
+                               Default: False.
+
+        # Page Navigation and Timing Parameters
+        wait_until (str): The condition to wait for when navigating, e.g. "domcontentloaded".
+                          Default: "domcontentloaded".
+        page_timeout (int): Timeout in ms for page operations like navigation.
+                            Default: 60000 (60 seconds).
+        wait_for (str or None): A CSS selector or JS condition to wait for before extracting content.
+                                Default: None.
+        wait_for_images (bool): If True, wait for images to load before extracting content.
+                                Default: False.
+        delay_before_return_html (float): Delay in seconds before retrieving final HTML.
+                                          Default: 0.1.
+        mean_delay (float): Mean base delay between requests when calling arun_many.
+                            Default: 0.1.
+        max_range (float): Max random additional delay range for requests in arun_many.
+                           Default: 0.3.
+        semaphore_count (int): Number of concurrent operations allowed.
+                               Default: 5.
+
+        # Page Interaction Parameters
+        js_code (str or list of str or None): JavaScript code/snippets to run on the page.
+                                              Default: None.
+        js_only (bool): If True, indicates subsequent calls are JS-driven updates, not full page loads.
+                        Default: False.
+        ignore_body_visibility (bool): If True, ignore whether the body is visible before proceeding.
+                                       Default: True.
+        scan_full_page (bool): If True, scroll through the entire page to load all content.
+                               Default: False.
+        scroll_delay (float): Delay in seconds between scroll steps if scan_full_page is True.
+                              Default: 0.2.
+        process_iframes (bool): If True, attempts to process and inline iframe content.
+                                Default: False.
+        remove_overlay_elements (bool): If True, remove overlays/popups before extracting HTML.
+                                        Default: False.
+        simulate_user (bool): If True, simulate user interactions (mouse moves, clicks) for anti-bot measures.
+                              Default: False.
+        override_navigator (bool): If True, overrides navigator properties for more human-like behavior.
+                                   Default: False.
+        magic (bool): If True, attempts automatic handling of overlays/popups.
+                      Default: False.
+        adjust_viewport_to_content (bool): If True, adjust viewport according to the page content dimensions.
+                                           Default: False.
+
+        # Media Handling Parameters
+        screenshot (bool): Whether to take a screenshot after crawling.
+                           Default: False.
+        screenshot_wait_for (float or None): Additional wait time before taking a screenshot.
+                                             Default: None.
+        screenshot_height_threshold (int): Threshold for page height to decide screenshot strategy.
+                                           Default: SCREENSHOT_HEIGHT_TRESHOLD (from config, e.g. 20000).
+        pdf (bool): Whether to generate a PDF of the page.
+                    Default: False.
+        image_description_min_word_threshold (int): Minimum words for image description extraction.
+                                                    Default: IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD (e.g., 50).
+        image_score_threshold (int): Minimum score threshold for processing an image.
+                                     Default: IMAGE_SCORE_THRESHOLD (e.g., 3).
+        exclude_external_images (bool): If True, exclude all external images from processing.
+                                         Default: False.
+
+        # Link and Domain Handling Parameters
+        exclude_social_media_domains (list of str): List of domains to exclude for social media links.
+                                                    Default: SOCIAL_MEDIA_DOMAINS (from config).
+        exclude_external_links (bool): If True, exclude all external links from the results.
+                                       Default: False.
+        exclude_social_media_links (bool): If True, exclude links pointing to social media domains.
+                                           Default: False.
+        exclude_domains (list of str): List of specific domains to exclude from results.
+                                       Default: [].
+
+        # Debugging and Logging Parameters
+        verbose (bool): Enable verbose logging.
+                        Default: True.
+        log_console (bool): If True, log console messages from the page.
+                            Default: False.
+    """
+
+    def __init__(
+        self,
+        # Content Processing Parameters
+        word_count_threshold: int = MIN_WORD_THRESHOLD,
+        extraction_strategy: ExtractionStrategy = None,
+        chunking_strategy: ChunkingStrategy = None,
+        markdown_generator: MarkdownGenerationStrategy = None,
+        content_filter=None,
+        only_text: bool = False,
+        css_selector: str = None,
+        excluded_tags: list = None,
+        excluded_selector: str = None,
+        keep_data_attributes: bool = False,
+        remove_forms: bool = False,
+        prettiify: bool = False,
+        parser_type: str = "lxml",
+
+        # SSL Parameters
+        fetch_ssl_certificate: bool = False,
+
+        # Caching Parameters
+        cache_mode=None,
+        session_id: str = None,
+        bypass_cache: bool = False,
+        disable_cache: bool = False,
+        no_cache_read: bool = False,
+        no_cache_write: bool = False,
+
+        # Page Navigation and Timing Parameters
+        wait_until: str = "domcontentloaded",
+        page_timeout: int = PAGE_TIMEOUT,
+        wait_for: str = None,
+        wait_for_images: bool = False,
+        delay_before_return_html: float = 0.1,
+        mean_delay: float = 0.1,
+        max_range: float = 0.3,
+        semaphore_count: int = 5,
+
+        # Page Interaction Parameters
+        js_code: Union[str, List[str]] = None,
+        js_only: bool = False,
+        ignore_body_visibility: bool = True,
+        scan_full_page: bool = False,
+        scroll_delay: float = 0.2,
+        process_iframes: bool = False,
+        remove_overlay_elements: bool = False,
+        simulate_user: bool = False,
+        override_navigator: bool = False,
+        magic: bool = False,
+        adjust_viewport_to_content: bool = False,
+
+        # Media Handling Parameters
+        screenshot: bool = False,
+        screenshot_wait_for: float = None,
+        screenshot_height_threshold: int = SCREENSHOT_HEIGHT_TRESHOLD,
+        pdf: bool = False,
+        image_description_min_word_threshold: int = IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD,
+        image_score_threshold: int = IMAGE_SCORE_THRESHOLD,
+        exclude_external_images: bool = False,
+
+        # Link and Domain Handling Parameters
+        exclude_social_media_domains: list = None,
+        exclude_external_links: bool = False,
+        exclude_social_media_links: bool = False,
+        exclude_domains: list = None,
+
+        # Debugging and Logging Parameters
+        verbose: bool = True,
+        log_console: bool = False,
+        
+        url: str = None,
+    ):
+        self.url = url
+        
+        # Content Processing Parameters
+        self.word_count_threshold = word_count_threshold
+        self.extraction_strategy = extraction_strategy
+        self.chunking_strategy = chunking_strategy
+        self.markdown_generator = markdown_generator
+        self.content_filter = content_filter
+        self.only_text = only_text
+        self.css_selector = css_selector
+        self.excluded_tags = excluded_tags or []
+        self.excluded_selector = excluded_selector or ""
+        self.keep_data_attributes = keep_data_attributes
+        self.remove_forms = remove_forms
+        self.prettiify = prettiify
+        self.parser_type = parser_type
+
+        # SSL Parameters
+        self.fetch_ssl_certificate = fetch_ssl_certificate
+
+        # Caching Parameters
+        self.cache_mode = cache_mode
+        self.session_id = session_id
+        self.bypass_cache = bypass_cache
+        self.disable_cache = disable_cache
+        self.no_cache_read = no_cache_read
+        self.no_cache_write = no_cache_write
+
+        # Page Navigation and Timing Parameters
+        self.wait_until = wait_until
+        self.page_timeout = page_timeout
+        self.wait_for = wait_for
+        self.wait_for_images = wait_for_images
+        self.delay_before_return_html = delay_before_return_html
+        self.mean_delay = mean_delay
+        self.max_range = max_range
+        self.semaphore_count = semaphore_count
+
+        # Page Interaction Parameters
+        self.js_code = js_code
+        self.js_only = js_only
+        self.ignore_body_visibility = ignore_body_visibility
+        self.scan_full_page = scan_full_page
+        self.scroll_delay = scroll_delay
+        self.process_iframes = process_iframes
+        self.remove_overlay_elements = remove_overlay_elements
+        self.simulate_user = simulate_user
+        self.override_navigator = override_navigator
+        self.magic = magic
+        self.adjust_viewport_to_content = adjust_viewport_to_content
+
+        # Media Handling Parameters
+        self.screenshot = screenshot
+        self.screenshot_wait_for = screenshot_wait_for
+        self.screenshot_height_threshold = screenshot_height_threshold
+        self.pdf = pdf
+        self.image_description_min_word_threshold = image_description_min_word_threshold
+        self.image_score_threshold = image_score_threshold
+        self.exclude_external_images = exclude_external_images
+
+        # Link and Domain Handling Parameters
+        self.exclude_social_media_domains = exclude_social_media_domains or SOCIAL_MEDIA_DOMAINS
+        self.exclude_external_links = exclude_external_links
+        self.exclude_social_media_links = exclude_social_media_links
+        self.exclude_domains = exclude_domains or []
+
+        # Debugging and Logging Parameters
+        self.verbose = verbose
+        self.log_console = log_console
+
+        # Validate type of extraction strategy and chunking strategy if they are provided
+        if self.extraction_strategy is not None and not isinstance(
+            self.extraction_strategy, ExtractionStrategy
+        ):
+            raise ValueError("extraction_strategy must be an instance of ExtractionStrategy")
+        if self.chunking_strategy is not None and not isinstance(
+            self.chunking_strategy, ChunkingStrategy
+        ):
+            raise ValueError("chunking_strategy must be an instance of ChunkingStrategy")
+
+        # Set default chunking strategy if None
+        if self.chunking_strategy is None:
+            from .chunking_strategy import RegexChunking
+            self.chunking_strategy = RegexChunking()
+
+    @staticmethod
+    def from_kwargs(kwargs: dict) -> "CrawlerRunConfig":
+        return CrawlerRunConfig(
+            # Content Processing Parameters
+            word_count_threshold=kwargs.get("word_count_threshold", 200),
+            extraction_strategy=kwargs.get("extraction_strategy"),
+            chunking_strategy=kwargs.get("chunking_strategy"),
+            markdown_generator=kwargs.get("markdown_generator"),
+            content_filter=kwargs.get("content_filter"),
+            only_text=kwargs.get("only_text", False),
+            css_selector=kwargs.get("css_selector"),
+            excluded_tags=kwargs.get("excluded_tags", []),
+            excluded_selector=kwargs.get("excluded_selector", ""),
+            keep_data_attributes=kwargs.get("keep_data_attributes", False),
+            remove_forms=kwargs.get("remove_forms", False),
+            prettiify=kwargs.get("prettiify", False),
+            parser_type=kwargs.get("parser_type", "lxml"),
+
+            # SSL Parameters
+            fetch_ssl_certificate=kwargs.get("fetch_ssl_certificate", False),
+
+            # Caching Parameters
+            cache_mode=kwargs.get("cache_mode"),
+            session_id=kwargs.get("session_id"),
+            bypass_cache=kwargs.get("bypass_cache", False),
+            disable_cache=kwargs.get("disable_cache", False),
+            no_cache_read=kwargs.get("no_cache_read", False),
+            no_cache_write=kwargs.get("no_cache_write", False),
+
+            # Page Navigation and Timing Parameters
+            wait_until=kwargs.get("wait_until", "domcontentloaded"),
+            page_timeout=kwargs.get("page_timeout", 60000),
+            wait_for=kwargs.get("wait_for"),
+            wait_for_images=kwargs.get("wait_for_images", False),
+            delay_before_return_html=kwargs.get("delay_before_return_html", 0.1),
+            mean_delay=kwargs.get("mean_delay", 0.1),
+            max_range=kwargs.get("max_range", 0.3),
+            semaphore_count=kwargs.get("semaphore_count", 5),
+
+            # Page Interaction Parameters
+            js_code=kwargs.get("js_code"),
+            js_only=kwargs.get("js_only", False),
+            ignore_body_visibility=kwargs.get("ignore_body_visibility", True),
+            scan_full_page=kwargs.get("scan_full_page", False),
+            scroll_delay=kwargs.get("scroll_delay", 0.2),
+            process_iframes=kwargs.get("process_iframes", False),
+            remove_overlay_elements=kwargs.get("remove_overlay_elements", False),
+            simulate_user=kwargs.get("simulate_user", False),
+            override_navigator=kwargs.get("override_navigator", False),
+            magic=kwargs.get("magic", False),
+            adjust_viewport_to_content=kwargs.get("adjust_viewport_to_content", False),
+
+            # Media Handling Parameters
+            screenshot=kwargs.get("screenshot", False),
+            screenshot_wait_for=kwargs.get("screenshot_wait_for"),
+            screenshot_height_threshold=kwargs.get("screenshot_height_threshold", SCREENSHOT_HEIGHT_TRESHOLD),
+            pdf=kwargs.get("pdf", False),
+            image_description_min_word_threshold=kwargs.get("image_description_min_word_threshold", IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD),
+            image_score_threshold=kwargs.get("image_score_threshold", IMAGE_SCORE_THRESHOLD),
+            exclude_external_images=kwargs.get("exclude_external_images", False),
+
+            # Link and Domain Handling Parameters
+            exclude_social_media_domains=kwargs.get("exclude_social_media_domains", SOCIAL_MEDIA_DOMAINS),
+            exclude_external_links=kwargs.get("exclude_external_links", False),
+            exclude_social_media_links=kwargs.get("exclude_social_media_links", False),
+            exclude_domains=kwargs.get("exclude_domains", []),
+
+            # Debugging and Logging Parameters
+            verbose=kwargs.get("verbose", True),
+            log_console=kwargs.get("log_console", False),
+            
+            url=kwargs.get("url"),
+        )
+        
+    # Create a funciton returns dict of the object
+    def to_dict(self):
+        return {
+            "word_count_threshold": self.word_count_threshold,
+            "extraction_strategy": self.extraction_strategy,
+            "chunking_strategy": self.chunking_strategy,
+            "markdown_generator": self.markdown_generator,
+            "content_filter": self.content_filter,
+            "only_text": self.only_text,
+            "css_selector": self.css_selector,
+            "excluded_tags": self.excluded_tags,
+            "excluded_selector": self.excluded_selector,
+            "keep_data_attributes": self.keep_data_attributes,
+            "remove_forms": self.remove_forms,
+            "prettiify": self.prettiify,
+            "parser_type": self.parser_type,
+            "fetch_ssl_certificate": self.fetch_ssl_certificate,
+            "cache_mode": self.cache_mode,
+            "session_id": self.session_id,
+            "bypass_cache": self.bypass_cache,
+            "disable_cache": self.disable_cache,
+            "no_cache_read": self.no_cache_read,
+            "no_cache_write": self.no_cache_write,
+            "wait_until": self.wait_until,
+            "page_timeout": self.page_timeout,
+            "wait_for": self.wait_for,
+            "wait_for_images": self.wait_for_images,
+            "delay_before_return_html": self.delay_before_return_html,
+            "mean_delay": self.mean_delay,
+            "max_range": self.max_range,
+            "semaphore_count": self.semaphore_count,
+            "js_code": self.js_code,
+            "js_only": self.js_only,
+            "ignore_body_visibility": self.ignore_body_visibility,
+            "scan_full_page": self.scan_full_page,
+            "scroll_delay": self.scroll_delay,
+            "process_iframes": self.process_iframes,
+            "remove_overlay_elements": self.remove_overlay_elements,
+            "simulate_user": self.simulate_user,
+            "override_navigator": self.override_navigator,
+            "magic": self.magic,
+            "adjust_viewport_to_content": self.adjust_viewport_to_content,
+            "screenshot": self.screenshot,
+            "screenshot_wait_for": self.screenshot_wait_for,
+            "screenshot_height_threshold": self.screenshot_height_threshold,
+            "pdf": self.pdf,
+            "image_description_min_word_threshold": self.image_description_min_word_threshold,
+            "image_score_threshold": self.image_score_threshold,
+            "exclude_external_images": self.exclude_external_images,
+            "exclude_social_media_domains": self.exclude_social_media_domains,
+            "exclude_external_links": self.exclude_external_links,
+            "exclude_social_media_links": self.exclude_social_media_links,
+            "exclude_domains": self.exclude_domains,
+            "verbose": self.verbose,
+            "log_console": self.log_console,
+            "url": self.url,
+        }

crawl4ai/async_crawler_strategy.py

@@ -0,0 +1,2191 @@
+import asyncio
+import base64
+import time
+from abc import ABC, abstractmethod
+from typing import Callable, Dict, Any, List, Optional, Awaitable, Union
+import os, sys, shutil
+import tempfile, subprocess
+from playwright.async_api import async_playwright, Page, Browser, Error, BrowserContext
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+from io import BytesIO
+from PIL import Image, ImageDraw, ImageFont
+from pathlib import Path
+from playwright.async_api import ProxySettings
+from pydantic import BaseModel
+import hashlib
+import json
+import uuid
+from .js_snippet import load_js_script
+from .models import AsyncCrawlResponse
+from .utils import get_error_context
+from .user_agent_generator import UserAgentGenerator
+from .config import SCREENSHOT_HEIGHT_TRESHOLD, DOWNLOAD_PAGE_TIMEOUT
+from .async_configs import BrowserConfig, CrawlerRunConfig
+from .async_logger import AsyncLogger
+from playwright_stealth import StealthConfig, stealth_async
+from .ssl_certificate import SSLCertificate
+
+stealth_config = StealthConfig(
+    webdriver=True,
+    chrome_app=True,
+    chrome_csi=True,
+    chrome_load_times=True,
+    chrome_runtime=True,
+    navigator_languages=True,
+    navigator_plugins=True,
+    navigator_permissions=True,
+    webgl_vendor=True,
+    outerdimensions=True,
+    navigator_hardware_concurrency=True,
+    media_codecs=True,
+)
+
+BROWSER_DISABLE_OPTIONS = [
+    "--disable-background-networking",
+    "--disable-background-timer-throttling",
+    "--disable-backgrounding-occluded-windows",
+    "--disable-breakpad",
+    "--disable-client-side-phishing-detection",
+    "--disable-component-extensions-with-background-pages",
+    "--disable-default-apps",
+    "--disable-extensions",
+    "--disable-features=TranslateUI",
+    "--disable-hang-monitor",
+    "--disable-ipc-flooding-protection",
+    "--disable-popup-blocking",
+    "--disable-prompt-on-repost",
+    "--disable-sync",
+    "--force-color-profile=srgb",
+    "--metrics-recording-only",
+    "--no-first-run",
+    "--password-store=basic",
+    "--use-mock-keychain",
+]
+
+
+class ManagedBrowser:
+    """
+    Manages the browser process and context. This class allows to connect to the browser using CDP protocol.
+    
+    Attributes:
+        browser_type (str): The type of browser to launch. Supported values: "chromium", "firefox", "webkit".
+                            Default: "chromium".
+        user_data_dir (str or None): Path to a user data directory for persistent sessions. If None, a
+                                     temporary directory may be used. Default: None.
+        headless (bool): Whether to run the browser in headless mode (no visible GUI).
+                         Default: True.
+        browser_process (subprocess.Popen): The process object for the browser.
+        temp_dir (str): Temporary directory for user data if not provided.  
+        debugging_port (int): Port for debugging the browser.
+        host (str): Host for debugging the browser.
+        
+        Methods:
+            start(): Starts the browser process and returns the CDP endpoint URL.
+            _get_browser_path(): Returns the browser executable path based on OS and browser type.
+            _get_browser_args(): Returns browser-specific command line arguments.
+            _get_user_data_dir(): Returns the user data directory path.
+            _cleanup(): Terminates the browser process and removes the temporary directory. 
+    """
+
+    browser_type: str
+    user_data_dir: str
+    headless: bool
+    browser_process: subprocess.Popen
+    temp_dir: str
+    debugging_port: int
+    host: str
+    def __init__(
+        self,
+        browser_type: str = "chromium",
+        user_data_dir: Optional[str] = None,
+        headless: bool = False,
+        logger=None,
+        host: str = "localhost",
+        debugging_port: int = 9222,
+    ):
+        """
+        Initialize the ManagedBrowser instance.
+        
+        Args:
+            browser_type (str): The type of browser to launch. Supported values: "chromium", "firefox", "webkit".
+                                Default: "chromium".
+            user_data_dir (str or None): Path to a user data directory for persistent sessions. If None, a
+                                         temporary directory may be used. Default: None.
+            headless (bool): Whether to run the browser in headless mode (no visible GUI).
+                             Default: True.
+            logger (logging.Logger): Logger instance for logging messages. Default: None.
+            host (str): Host for debugging the browser. Default: "localhost".
+            debugging_port (int): Port for debugging the browser. Default: 9222.
+        """ 
+        self.browser_type = browser_type
+        self.user_data_dir = user_data_dir
+        self.headless = headless
+        self.browser_process = None
+        self.temp_dir = None
+        self.debugging_port = debugging_port
+        self.host = host
+        self.logger = logger
+        self.shutting_down = False
+
+    async def start(self) -> str:
+        """
+        Starts the browser process and returns the CDP endpoint URL.
+        If user_data_dir is not provided, creates a temporary directory.
+        """
+
+        # Create temp dir if needed
+        if not self.user_data_dir:
+            self.temp_dir = tempfile.mkdtemp(prefix="browser-profile-")
+            self.user_data_dir = self.temp_dir
+
+        # Get browser path and args based on OS and browser type
+        browser_path = self._get_browser_path()
+        args = self._get_browser_args()
+
+        # Start browser process
+        try:
+            self.browser_process = subprocess.Popen(
+                args, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+            )
+            # Monitor browser process output for errors
+            asyncio.create_task(self._monitor_browser_process())
+            await asyncio.sleep(2)  # Give browser time to start
+            return f"http://{self.host}:{self.debugging_port}"
+        except Exception as e:
+            await self.cleanup()
+            raise Exception(f"Failed to start browser: {e}")
+
+    async def _monitor_browser_process(self):
+        """
+        Monitor the browser process for unexpected termination.
+        
+        How it works:
+        1. Read stdout and stderr from the browser process.
+        2. If the process has terminated, log the error message and terminate the browser.
+        3. If the shutting_down flag is set, log the normal termination message.
+        4. If any other error occurs, log the error message.
+        
+        Note: This method should be called in a separate task to avoid blocking the main event loop.
+        """
+        if self.browser_process:
+            try:
+                stdout, stderr = await asyncio.gather(
+                    asyncio.to_thread(self.browser_process.stdout.read),
+                    asyncio.to_thread(self.browser_process.stderr.read),
+                )
+
+                # Check shutting_down flag BEFORE logging anything
+                if self.browser_process.poll() is not None:
+                    if not self.shutting_down:
+                        self.logger.error(
+                            message="Browser process terminated unexpectedly | Code: {code} | STDOUT: {stdout} | STDERR: {stderr}",
+                            tag="ERROR",
+                            params={
+                                "code": self.browser_process.returncode,
+                                "stdout": stdout.decode(),
+                                "stderr": stderr.decode(),
+                            },
+                        )
+                        await self.cleanup()
+                    else:
+                        self.logger.info(
+                            message="Browser process terminated normally | Code: {code}",
+                            tag="INFO",
+                            params={"code": self.browser_process.returncode},
+                        )
+            except Exception as e:
+                if not self.shutting_down:
+                    self.logger.error(
+                        message="Error monitoring browser process: {error}",
+                        tag="ERROR",
+                        params={"error": str(e)},
+                    )
+
+    def _get_browser_path(self) -> str:
+        """Returns the browser executable path based on OS and browser type"""
+        if sys.platform == "darwin":  # macOS
+            paths = {
+                "chromium": "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+                "firefox": "/Applications/Firefox.app/Contents/MacOS/firefox",
+                "webkit": "/Applications/Safari.app/Contents/MacOS/Safari",
+            }
+        elif sys.platform == "win32":  # Windows
+            paths = {
+                "chromium": "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
+                "firefox": "C:\\Program Files\\Mozilla Firefox\\firefox.exe",
+                "webkit": None,  # WebKit not supported on Windows
+            }
+        else:  # Linux
+            paths = {
+                "chromium": "google-chrome",
+                "firefox": "firefox",
+                "webkit": None,  # WebKit not supported on Linux
+            }
+
+        return paths.get(self.browser_type)
+
+    def _get_browser_args(self) -> List[str]:
+        """Returns browser-specific command line arguments"""
+        base_args = [self._get_browser_path()]
+
+        if self.browser_type == "chromium":
+            args = [
+                f"--remote-debugging-port={self.debugging_port}",
+                f"--user-data-dir={self.user_data_dir}",
+            ]
+            if self.headless:
+                args.append("--headless=new")
+        elif self.browser_type == "firefox":
+            args = [
+                "--remote-debugging-port",
+                str(self.debugging_port),
+                "--profile",
+                self.user_data_dir,
+            ]
+            if self.headless:
+                args.append("--headless")
+        else:
+            raise NotImplementedError(f"Browser type {self.browser_type} not supported")
+
+        return base_args + args
+
+    async def cleanup(self):
+        """Cleanup browser process and temporary directory"""
+        # Set shutting_down flag BEFORE any termination actions
+        self.shutting_down = True
+
+        if self.browser_process:
+            try:
+                self.browser_process.terminate()
+                # Wait for process to end gracefully
+                for _ in range(10):  # 10 attempts, 100ms each
+                    if self.browser_process.poll() is not None:
+                        break
+                    await asyncio.sleep(0.1)
+
+                # Force kill if still running
+                if self.browser_process.poll() is None:
+                    self.browser_process.kill()
+                    await asyncio.sleep(0.1)  # Brief wait for kill to take effect
+
+            except Exception as e:
+                self.logger.error(
+                    message="Error terminating browser: {error}",
+                    tag="ERROR",
+                    params={"error": str(e)},
+                )
+
+        if self.temp_dir and os.path.exists(self.temp_dir):
+            try:
+                shutil.rmtree(self.temp_dir)
+            except Exception as e:
+                self.logger.error(
+                    message="Error removing temporary directory: {error}",
+                    tag="ERROR",
+                    params={"error": str(e)},
+                )
+
+
+class BrowserManager:
+    """
+    Manages the browser instance and context.
+    
+    Attributes: 
+        config (BrowserConfig): Configuration object containing all browser settings
+        logger: Logger instance for recording events and errors
+        browser (Browser): The browser instance
+        default_context (BrowserContext): The default browser context    
+        managed_browser (ManagedBrowser): The managed browser instance
+        playwright (Playwright): The Playwright instance
+        sessions (dict): Dictionary to store session information
+        session_ttl (int): Session timeout in seconds
+    """
+    def __init__(self, browser_config: BrowserConfig, logger=None):
+        """
+        Initialize the BrowserManager with a browser configuration.
+
+        Args:
+            browser_config (BrowserConfig): Configuration object containing all browser settings
+            logger: Logger instance for recording events and errors
+        """
+        self.config: BrowserConfig = browser_config
+        self.logger = logger
+
+        # Browser state
+        self.browser = None
+        self.default_context = None
+        self.managed_browser = None
+        self.playwright = None
+
+        # Session management
+        self.sessions = {}
+        self.session_ttl = 1800  # 30 minutes
+
+        # Initialize ManagedBrowser if needed
+        if self.config.use_managed_browser:
+            self.managed_browser = ManagedBrowser(
+                browser_type=self.config.browser_type,
+                user_data_dir=self.config.user_data_dir,
+                headless=self.config.headless,
+                logger=self.logger,
+                debugging_port=self.config.debugging_port,
+            )
+
+    async def start(self):
+        """
+        Start the browser instance and set up the default context.
+        
+        How it works:
+        1. Check if Playwright is already initialized.
+        2. If not, initialize Playwright.
+        3. If managed browser is used, start it and connect to the CDP endpoint.
+        4. If managed browser is not used, launch the browser and set up the default context.
+        
+        Note: This method should be called in a separate task to avoid blocking the main event loop.
+        """
+        if self.playwright is None:
+            from playwright.async_api import async_playwright
+
+            self.playwright = await async_playwright().start()
+
+        if self.config.use_managed_browser:
+            cdp_url = await self.managed_browser.start()
+            self.browser = await self.playwright.chromium.connect_over_cdp(cdp_url)
+            contexts = self.browser.contexts
+            if contexts:
+                self.default_context = contexts[0]
+            else:
+                self.default_context = await self.create_browser_context()
+                # self.default_context = await self.browser.new_context(
+                #     viewport={
+                #         "width": self.config.viewport_width,
+                #         "height": self.config.viewport_height,
+                #     },
+                #     storage_state=self.config.storage_state,
+                #     user_agent=self.config.headers.get(
+                #         "User-Agent", self.config.user_agent
+                #     ),
+                #     accept_downloads=self.config.accept_downloads,
+                #     ignore_https_errors=self.config.ignore_https_errors,
+                #     java_script_enabled=self.config.java_script_enabled,
+                # )
+            await self.setup_context(self.default_context)
+        else:
+            browser_args = self._build_browser_args()
+
+            # Launch appropriate browser type
+            if self.config.browser_type == "firefox":
+                self.browser = await self.playwright.firefox.launch(**browser_args)
+            elif self.config.browser_type == "webkit":
+                self.browser = await self.playwright.webkit.launch(**browser_args)
+            else:
+                self.browser = await self.playwright.chromium.launch(**browser_args)
+
+            self.default_context = self.browser
+
+    def _build_browser_args(self) -> dict:
+        """Build browser launch arguments from config."""
+        args = [
+            "--disable-gpu",
+            "--disable-gpu-compositing",
+            "--disable-software-rasterizer",
+            "--no-sandbox",
+            "--disable-dev-shm-usage",
+            "--no-first-run",
+            "--no-default-browser-check",
+            "--disable-infobars",
+            "--window-position=0,0",
+            "--ignore-certificate-errors",
+            "--ignore-certificate-errors-spki-list",
+            "--disable-blink-features=AutomationControlled",
+            "--window-position=400,0",
+            "--disable-renderer-backgrounding",
+            "--disable-ipc-flooding-protection",
+            "--force-color-profile=srgb",
+            "--mute-audio",
+            "--disable-background-timer-throttling",
+            # "--single-process",
+            f"--window-size={self.config.viewport_width},{self.config.viewport_height}",
+        ]
+
+        if self.config.light_mode:
+            args.extend(BROWSER_DISABLE_OPTIONS)
+
+        if self.config.text_mode:
+            args.extend(
+                [
+                    "--blink-settings=imagesEnabled=false",
+                    "--disable-remote-fonts",
+                    "--disable-images",
+                    "--disable-javascript",
+                    "--disable-software-rasterizer",
+                    "--disable-dev-shm-usage",
+                ]
+            )
+
+        if self.config.extra_args:
+            args.extend(self.config.extra_args)
+
+        browser_args = {"headless": self.config.headless, "args": args}
+
+        if self.config.chrome_channel:
+            browser_args["channel"] = self.config.chrome_channel
+
+        if self.config.accept_downloads:
+            browser_args["downloads_path"] = self.config.downloads_path or os.path.join(
+                os.getcwd(), "downloads"
+            )
+            os.makedirs(browser_args["downloads_path"], exist_ok=True)
+
+        if self.config.proxy or self.config.proxy_config:
+            from playwright.async_api import ProxySettings
+
+            proxy_settings = (
+                ProxySettings(server=self.config.proxy)
+                if self.config.proxy
+                else ProxySettings(
+                    server=self.config.proxy_config.get("server"),
+                    username=self.config.proxy_config.get("username"),
+                    password=self.config.proxy_config.get("password"),
+                )
+            )
+            browser_args["proxy"] = proxy_settings
+
+        return browser_args
+
+    async def setup_context(
+        self,
+        context: BrowserContext,
+        crawlerRunConfig: CrawlerRunConfig,
+        is_default=False,
+    ):
+        """
+        Set up a browser context with the configured options.
+
+        How it works:
+        1. Set extra HTTP headers if provided.
+        2. Add cookies if provided.
+        3. Load storage state if provided.
+        4. Accept downloads if enabled.
+        5. Set default timeouts for navigation and download.
+        6. Set user agent if provided.
+        7. Set browser hints if provided.
+        8. Set proxy if provided.
+        9. Set downloads path if provided.
+        10. Set storage state if provided.
+        11. Set cache if provided.
+        12. Set extra HTTP headers if provided.
+        13. Add cookies if provided.
+        14. Set default timeouts for navigation and download if enabled.
+        15. Set user agent if provided.
+        16. Set browser hints if provided.
+        
+        Args:
+            context (BrowserContext): The browser context to set up
+            crawlerRunConfig (CrawlerRunConfig): Configuration object containing all browser settings
+            is_default (bool): Flag indicating if this is the default context        
+        Returns:
+            None
+        """
+        if self.config.headers:
+            await context.set_extra_http_headers(self.config.headers)
+
+        if self.config.cookies:
+            await context.add_cookies(self.config.cookies)
+
+        if self.config.storage_state:
+            await context.storage_state(path=None)
+
+        if self.config.accept_downloads:
+            context.set_default_timeout(DOWNLOAD_PAGE_TIMEOUT)
+            context.set_default_navigation_timeout(DOWNLOAD_PAGE_TIMEOUT)
+            if self.config.downloads_path:
+                context._impl_obj._options["accept_downloads"] = True
+                context._impl_obj._options["downloads_path"] = (
+                    self.config.downloads_path
+                )
+
+        # Handle user agent and browser hints
+        if self.config.user_agent:
+            combined_headers = {
+                "User-Agent": self.config.user_agent,
+                "sec-ch-ua": self.config.browser_hint,
+            }
+            combined_headers.update(self.config.headers)
+            await context.set_extra_http_headers(combined_headers)
+
+        # Add default cookie
+        await context.add_cookies(
+            [{"name": "cookiesEnabled", "value": "true", "url": crawlerRunConfig.url}]
+        )
+
+        # Handle navigator overrides
+        if (
+            crawlerRunConfig.override_navigator
+            or crawlerRunConfig.simulate_user
+            or crawlerRunConfig.magic
+        ):
+            await context.add_init_script(load_js_script("navigator_overrider"))
+
+    async def create_browser_context(self):
+        """
+        Creates and returns a new browser context with configured settings.
+        Applies text-only mode settings if text_mode is enabled in config.
+        
+        Returns:
+            Context: Browser context object with the specified configurations
+        """
+        # Base settings
+        user_agent = self.config.headers.get("User-Agent", self.config.user_agent)
+        viewport_settings = {
+            "width": self.config.viewport_width,
+            "height": self.config.viewport_height,
+        }
+        proxy_settings = {"server": self.config.proxy} if self.config.proxy else None
+        
+        blocked_extensions = [
+            # Images
+            'jpg', 'jpeg', 'png', 'gif', 'webp', 'svg', 'ico', 'bmp', 'tiff', 'psd',
+            # Fonts
+            'woff', 'woff2', 'ttf', 'otf', 'eot',
+            # Styles
+            # 'css', 'less', 'scss', 'sass',
+            # Media
+            'mp4', 'webm', 'ogg', 'avi', 'mov', 'wmv', 'flv', 'm4v',
+            'mp3', 'wav', 'aac', 'm4a', 'opus', 'flac',
+            # Documents
+            'pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx',
+            # Archives
+            'zip', 'rar', '7z', 'tar', 'gz',
+            # Scripts and data
+            'xml', 'swf', 'wasm'
+        ]
+        
+        # Common context settings
+        context_settings = {
+            "user_agent": user_agent,
+            "viewport": viewport_settings,
+            "proxy": proxy_settings,
+            "accept_downloads": self.config.accept_downloads,
+            "storage_state": self.config.storage_state,
+            "ignore_https_errors": self.config.ignore_https_errors,
+            "device_scale_factor": 1.0,
+            "java_script_enabled": self.config.java_script_enabled,
+        }
+        
+        if self.config.text_mode:
+            text_mode_settings = {
+                "has_touch": False,
+                "is_mobile": False,
+            }
+            # Update context settings with text mode settings
+            context_settings.update(text_mode_settings)
+            
+        # Create and return the context with all settings
+        context = await self.browser.new_context(**context_settings)
+        
+        # Apply text mode settings if enabled
+        if self.config.text_mode:
+            # Create and apply route patterns for each extension
+            for ext in blocked_extensions:
+                await context.route(f"**/*.{ext}", lambda route: route.abort())
+        return context
+    
+    # async def get_page(self, session_id: Optional[str], user_agent: str):
+    async def get_page(self, crawlerRunConfig: CrawlerRunConfig):
+        """
+        Get a page for the given session ID, creating a new one if needed.
+        
+        Args:
+            crawlerRunConfig (CrawlerRunConfig): Configuration object containing all browser settings
+
+        Returns:
+            Page: The page object for the given session ID.
+            BrowserContext: The browser context for the given session ID.
+        """
+        self._cleanup_expired_sessions()
+
+        if crawlerRunConfig.session_id and crawlerRunConfig.session_id in self.sessions:
+            context, page, _ = self.sessions[crawlerRunConfig.session_id]
+            self.sessions[crawlerRunConfig.session_id] = (context, page, time.time())
+            return page, context
+
+        if self.config.use_managed_browser:
+            context = self.default_context
+            page = await context.new_page()
+        else:
+            context = await self.create_browser_context()
+            await self.setup_context(context, crawlerRunConfig)
+            page = await context.new_page()
+
+        if crawlerRunConfig.session_id:
+            self.sessions[crawlerRunConfig.session_id] = (context, page, time.time())
+
+        return page, context
+
+    async def kill_session(self, session_id: str):
+        """
+        Kill a browser session and clean up resources.  
+        
+        Args:
+            session_id (str): The session ID to kill.
+        """
+        if session_id in self.sessions:
+            context, page, _ = self.sessions[session_id]
+            await page.close()
+            if not self.config.use_managed_browser:
+                await context.close()
+            del self.sessions[session_id]
+
+    def _cleanup_expired_sessions(self):
+        """Clean up expired sessions based on TTL."""
+        current_time = time.time()
+        expired_sessions = [
+            sid
+            for sid, (_, _, last_used) in self.sessions.items()
+            if current_time - last_used > self.session_ttl
+        ]
+        for sid in expired_sessions:
+            asyncio.create_task(self.kill_session(sid))
+
+    async def close(self):
+        """Close all browser resources and clean up."""
+        if self.config.sleep_on_close:
+            await asyncio.sleep(0.5)
+
+        session_ids = list(self.sessions.keys())
+        for session_id in session_ids:
+            await self.kill_session(session_id)
+
+        if self.browser:
+            await self.browser.close()
+            self.browser = None
+
+        if self.managed_browser:
+            await asyncio.sleep(0.5)
+            await self.managed_browser.cleanup()
+            self.managed_browser = None
+
+        if self.playwright:
+            await self.playwright.stop()
+            self.playwright = None
+
+
+class AsyncCrawlerStrategy(ABC):
+    """
+    Abstract base class for crawler strategies.
+    Subclasses must implement the crawl method.
+    """
+    @abstractmethod
+    async def crawl(self, url: str, **kwargs) -> AsyncCrawlResponse:
+        pass  # 4 + 3
+
+
+
+class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
+    """
+    Crawler strategy using Playwright.
+    
+    Attributes:
+        browser_config (BrowserConfig): Configuration object containing browser settings.
+        logger (AsyncLogger): Logger instance for recording events and errors.
+        _downloaded_files (List[str]): List of downloaded file paths.   
+        hooks (Dict[str, Callable]): Dictionary of hooks for custom behavior.
+        browser_manager (BrowserManager): Manager for browser creation and management.
+
+        Methods:
+            __init__(self, browser_config=None, logger=None, **kwargs):
+                Initialize the AsyncPlaywrightCrawlerStrategy with a browser configuration.
+            __aenter__(self):
+                Start the browser and initialize the browser manager.
+            __aexit__(self, exc_type, exc_val, exc_tb):
+                Close the browser and clean up resources.
+            start(self):
+                Start the browser and initialize the browser manager.
+            close(self):
+                Close the browser and clean up resources.
+            kill_session(self, session_id):
+                Kill a browser session and clean up resources.
+            crawl(self, url, **kwargs):
+                Run the crawler for a single URL.
+            
+    """
+    def __init__(
+        self, browser_config: BrowserConfig = None, logger: AsyncLogger = None, **kwargs
+    ):
+        """
+        Initialize the AsyncPlaywrightCrawlerStrategy with a browser configuration.
+
+        Args:
+            browser_config (BrowserConfig): Configuration object containing browser settings.
+                                          If None, will be created from kwargs for backwards compatibility.
+            logger: Logger instance for recording events and errors.
+            **kwargs: Additional arguments for backwards compatibility and extending functionality.
+        """
+        # Initialize browser config, either from provided object or kwargs
+        self.browser_config = browser_config or BrowserConfig.from_kwargs(kwargs)
+        self.logger = logger
+
+        # Initialize session management
+        self._downloaded_files = []
+
+        # Initialize hooks system
+        self.hooks = {
+            "on_browser_created": None,
+            "on_page_context_created": None,
+            "on_user_agent_updated": None,
+            "on_execution_started": None,
+            "before_goto": None,
+            "after_goto": None,
+            "before_return_html": None,
+            "before_retrieve_html": None,
+        }
+
+        # Initialize browser manager with config
+        self.browser_manager = BrowserManager(
+            browser_config=self.browser_config, logger=self.logger
+        )
+
+    async def __aenter__(self):
+        await self.start()
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self.close()
+
+    async def start(self):
+        """
+        Start the browser and initialize the browser manager.
+        """
+        await self.browser_manager.start()
+        await self.execute_hook(
+            "on_browser_created",
+            self.browser_manager.browser,
+            context=self.browser_manager.default_context,
+        )
+
+    async def close(self):
+        """
+        Close the browser and clean up resources.
+        """
+        await self.browser_manager.close()
+
+    async def kill_session(self, session_id: str):
+        """
+        Kill a browser session and clean up resources.
+        
+        Args:
+            session_id (str): The ID of the session to kill.
+            
+        Returns:
+            None
+        """
+        # Log a warning message and no need kill session, in new version auto kill session
+        self.logger.warning(
+            message="Session auto-kill is enabled in the new version. No need to manually kill sessions.",
+            tag="WARNING",
+        )
+        await self.browser_manager.kill_session(session_id)
+
+    def set_hook(self, hook_type: str, hook: Callable):
+        """
+        Set a hook function for a specific hook type. Following are list of hook types:
+        - on_browser_created: Called when a new browser instance is created.
+        - on_page_context_created: Called when a new page context is created.    
+        - on_user_agent_updated: Called when the user agent is updated.    
+        - on_execution_started: Called when the execution starts.    
+        - before_goto: Called before a goto operation.    
+        - after_goto: Called after a goto operation.    
+        - before_return_html: Called before returning HTML content.    
+        - before_retrieve_html: Called before retrieving HTML content.  
+        
+        All hooks except on_browser_created accepts a context and a page as arguments and **kwargs. However, on_browser_created accepts a browser and a context as arguments and **kwargs.
+        
+        Args:
+            hook_type (str): The type of the hook.
+            hook (Callable): The hook function to set.
+            
+        Returns:
+            None
+        """
+        if hook_type in self.hooks:
+            self.hooks[hook_type] = hook
+        else:
+            raise ValueError(f"Invalid hook type: {hook_type}")
+
+    async def execute_hook(self, hook_type: str, *args, **kwargs):
+        """
+        Execute a hook function for a specific hook type.
+        
+        Args:
+            hook_type (str): The type of the hook.
+            *args: Variable length positional arguments.
+            **kwargs: Keyword arguments.
+            
+        Returns:
+            The return value of the hook function, if any.
+        """
+        hook = self.hooks.get(hook_type)
+        if hook:
+            if asyncio.iscoroutinefunction(hook):
+                return await hook(*args, **kwargs)
+            else:
+                return hook(*args, **kwargs)
+        return args[0] if args else None
+
+    def update_user_agent(self, user_agent: str):
+        """
+        Update the user agent for the browser.
+        
+        Args:
+            user_agent (str): The new user agent string.
+            
+        Returns:
+            None
+        """
+        self.user_agent = user_agent
+
+    def set_custom_headers(self, headers: Dict[str, str]):
+        """ 
+        Set custom headers for the browser. 
+        
+        Args:
+            headers (Dict[str, str]): A dictionary of headers to set.
+            
+        Returns:
+            None
+        """
+        self.headers = headers
+
+    async def smart_wait(self, page: Page, wait_for: str, timeout: float = 30000):
+        """ 
+        Wait for a condition in a smart way. This functions works as below:
+        
+        1. If wait_for starts with 'js:', it assumes it's a JavaScript function and waits for it to return true.
+        2. If wait_for starts with 'css:', it assumes it's a CSS selector and waits for it to be present.
+        3. Otherwise, it tries to evaluate wait_for as a JavaScript function and waits for it to return true.
+        4. If it's not a JavaScript function, it assumes it's a CSS selector and waits for it to be present.
+        
+        This is a more advanced version of the wait_for parameter in CrawlerStrategy.crawl().        
+        Args:
+            page: Playwright page object
+            wait_for (str): The condition to wait for. Can be a CSS selector, a JavaScript function, or explicitly prefixed with 'js:' or 'css:'.
+            timeout (float): Maximum time to wait in milliseconds
+            
+        Returns:
+            None
+        """
+        wait_for = wait_for.strip()
+
+        if wait_for.startswith("js:"):
+            # Explicitly specified JavaScript
+            js_code = wait_for[3:].strip()
+            return await self.csp_compliant_wait(page, js_code, timeout)
+        elif wait_for.startswith("css:"):
+            # Explicitly specified CSS selector
+            css_selector = wait_for[4:].strip()
+            try:
+                await page.wait_for_selector(css_selector, timeout=timeout)
+            except Error as e:
+                if "Timeout" in str(e):
+                    raise TimeoutError(
+                        f"Timeout after {timeout}ms waiting for selector '{css_selector}'"
+                    )
+                else:
+                    raise ValueError(f"Invalid CSS selector: '{css_selector}'")
+        else:
+            # Auto-detect based on content
+            if wait_for.startswith("()") or wait_for.startswith("function"):
+                # It's likely a JavaScript function
+                return await self.csp_compliant_wait(page, wait_for, timeout)
+            else:
+                # Assume it's a CSS selector first
+                try:
+                    await page.wait_for_selector(wait_for, timeout=timeout)
+                except Error as e:
+                    if "Timeout" in str(e):
+                        raise TimeoutError(
+                            f"Timeout after {timeout}ms waiting for selector '{wait_for}'"
+                        )
+                    else:
+                        # If it's not a timeout error, it might be an invalid selector
+                        # Let's try to evaluate it as a JavaScript function as a fallback
+                        try:
+                            return await self.csp_compliant_wait(
+                                page, f"() => {{{wait_for}}}", timeout
+                            )
+                        except Error:
+                            raise ValueError(
+                                f"Invalid wait_for parameter: '{wait_for}'. "
+                                "It should be either a valid CSS selector, a JavaScript function, "
+                                "or explicitly prefixed with 'js:' or 'css:'."
+                            )
+
+    async def csp_compliant_wait( self, page: Page, user_wait_function: str, timeout: float = 30000 ):
+        """
+        Wait for a condition in a CSP-compliant way.
+        
+        Args:
+            page: Playwright page object
+            user_wait_function: JavaScript function as string that returns boolean
+            timeout: Maximum time to wait in milliseconds
+            
+        Returns:
+            bool: True if condition was met, False if timed out
+            
+        Raises:
+            RuntimeError: If there's an error evaluating the condition
+        """
+        wrapper_js = f"""
+        async () => {{
+            const userFunction = {user_wait_function};
+            const startTime = Date.now();
+            try {{
+                while (true) {{
+                    if (await userFunction()) {{
+                        return true;
+                    }}
+                    if (Date.now() - startTime > {timeout}) {{
+                        return false;  // Return false instead of throwing
+                    }}
+                    await new Promise(resolve => setTimeout(resolve, 100));
+                }}
+            }} catch (error) {{
+                throw new Error(`Error evaluating condition: ${{error.message}}`);
+            }}
+        }}
+        """
+
+        try:
+            result = await page.evaluate(wrapper_js)
+            return result
+        except Exception as e:
+            if "Error evaluating condition" in str(e):
+                raise RuntimeError(f"Failed to evaluate wait condition: {str(e)}")
+            # For timeout or other cases, just return False
+            return False
+
+    async def process_iframes(self, page):
+        """
+        Process iframes on a page. This function will extract the content of each iframe and replace it with a div containing the extracted content.
+        
+        Args:
+            page: Playwright page object
+            
+        Returns:
+            Playwright page object
+        """
+        # Find all iframes
+        iframes = await page.query_selector_all("iframe")
+
+        for i, iframe in enumerate(iframes):
+            try:
+                # Add a unique identifier to the iframe
+                await iframe.evaluate(f'(element) => element.id = "iframe-{i}"')
+
+                # Get the frame associated with this iframe
+                frame = await iframe.content_frame()
+
+                if frame:
+                    # Wait for the frame to load
+                    await frame.wait_for_load_state(
+                        "load", timeout=30000
+                    )  # 30 seconds timeout
+
+                    # Extract the content of the iframe's body
+                    iframe_content = await frame.evaluate(
+                        "() => document.body.innerHTML"
+                    )
+
+                    # Generate a unique class name for this iframe
+                    class_name = f"extracted-iframe-content-{i}"
+
+                    # Replace the iframe with a div containing the extracted content
+                    _iframe = iframe_content.replace("`", "\\`")
+                    await page.evaluate(
+                        f"""
+                        () => {{
+                            const iframe = document.getElementById('iframe-{i}');
+                            const div = document.createElement('div');
+                            div.innerHTML = `{_iframe}`;
+                            div.className = '{class_name}';
+                            iframe.replaceWith(div);
+                        }}
+                    """
+                    )
+                else:
+                    self.logger.warning(
+                        message="Could not access content frame for iframe {index}",
+                        tag="SCRAPE",
+                        params={"index": i},
+                    )
+            except Exception as e:
+                self.logger.error(
+                    message="Error processing iframe {index}: {error}",
+                    tag="ERROR",
+                    params={"index": i, "error": str(e)},
+                )
+
+        # Return the page object
+        return page
+
+    async def create_session(self, **kwargs) -> str:
+        """
+        Creates a new browser session and returns its ID. A browse session is a unique openned page can be reused for multiple crawls.
+        This function is asynchronous and returns a string representing the session ID.
+        
+        Args:
+            **kwargs: Optional keyword arguments to configure the session.
+        
+        Returns:
+            str: The session ID.
+        """
+        await self.start()
+
+        session_id = kwargs.get("session_id") or str(uuid.uuid4())
+
+        user_agent = kwargs.get("user_agent", self.user_agent)
+        # Use browser_manager to get a fresh page & context assigned to this session_id
+        page, context = await self.browser_manager.get_page(session_id, user_agent)
+        return session_id
+
+    async def crawl( self, url: str, config: CrawlerRunConfig, **kwargs ) -> AsyncCrawlResponse:
+        """
+        Crawls a given URL or processes raw HTML/local file content based on the URL prefix.
+
+        Args:
+            url (str): The URL to crawl. Supported prefixes:
+                - 'http://' or 'https://': Web URL to crawl.
+                - 'file://': Local file path to process.
+                - 'raw://': Raw HTML content to process.
+            **kwargs: Additional parameters:
+                - 'screenshot' (bool): Whether to take a screenshot.
+                - ... [other existing parameters]
+
+        Returns:
+            AsyncCrawlResponse: The response containing HTML, headers, status code, and optional screenshot.
+        """
+        config = config or CrawlerRunConfig.from_kwargs(kwargs)
+        response_headers = {}
+        status_code = 200  # Default for local/raw HTML
+        screenshot_data = None
+
+        if url.startswith(("http://", "https://")):
+            return await self._crawl_web(url, config)
+
+        elif url.startswith("file://"):
+            # Process local file
+            local_file_path = url[7:]  # Remove 'file://' prefix
+            if not os.path.exists(local_file_path):
+                raise FileNotFoundError(f"Local file not found: {local_file_path}")
+            with open(local_file_path, "r", encoding="utf-8") as f:
+                html = f.read()
+            if config.screenshot:
+                screenshot_data = await self._generate_screenshot_from_html(html)
+            return AsyncCrawlResponse(
+                html=html,
+                response_headers=response_headers,
+                status_code=status_code,
+                screenshot=screenshot_data,
+                get_delayed_content=None,
+            )
+
+        elif url.startswith("raw:") or url.startswith("raw://"):
+            # Process raw HTML content
+            raw_html = url[4:] if url[:4] == "raw:" else url[7:]
+            html = raw_html
+            if config.screenshot:
+                screenshot_data = await self._generate_screenshot_from_html(html)
+            return AsyncCrawlResponse(
+                html=html,
+                response_headers=response_headers,
+                status_code=status_code,
+                screenshot=screenshot_data,
+                get_delayed_content=None,
+            )
+        else:
+            raise ValueError(
+                "URL must start with 'http://', 'https://', 'file://', or 'raw:'"
+            )
+
+    async def _crawl_web( self, url: str, config: CrawlerRunConfig ) -> AsyncCrawlResponse:
+        """
+        Internal method to crawl web URLs with the specified configuration.
+
+        Args:
+            url (str): The web URL to crawl
+            config (CrawlerRunConfig): Configuration object controlling the crawl behavior
+
+        Returns:
+            AsyncCrawlResponse: The response containing HTML, headers, status code, and optional data
+        """
+        config.url = url
+        response_headers = {}
+        status_code = None
+
+        # Reset downloaded files list for new crawl
+        self._downloaded_files = []
+
+        # Handle user agent with magic mode
+        user_agent = self.browser_config.user_agent
+        if config.magic and self.browser_config.user_agent_mode != "random":
+            self.browser_config.user_agent = UserAgentGenerator().generate(
+                **(self.browser_config.user_agent_generator_config or {})
+            )
+
+        # Get page for session
+        page, context = await self.browser_manager.get_page(crawlerRunConfig=config)
+
+        # Add default cookie
+        await context.add_cookies(
+            [{"name": "cookiesEnabled", "value": "true", "url": url}]
+        )
+
+        # Handle navigator overrides
+        if config.override_navigator or config.simulate_user or config.magic:
+            await context.add_init_script(load_js_script("navigator_overrider"))
+
+        # Call hook after page creation
+        await self.execute_hook("on_page_context_created", page, context=context)
+
+        # Set up console logging if requested
+        if config.log_console:
+
+            def log_consol(
+                msg, console_log_type="debug"
+            ):  # Corrected the parameter syntax
+                if console_log_type == "error":
+                    self.logger.error(
+                        message=f"Console error: {msg}",  # Use f-string for variable interpolation
+                        tag="CONSOLE",
+                        params={"msg": msg.text},
+                    )
+                elif console_log_type == "debug":
+                    self.logger.debug(
+                        message=f"Console: {msg}",  # Use f-string for variable interpolation
+                        tag="CONSOLE",
+                        params={"msg": msg.text},
+                    )
+
+            page.on("console", log_consol)
+            page.on("pageerror", lambda e: log_consol(e, "error"))
+
+        try:
+            # Get SSL certificate information if requested and URL is HTTPS
+            ssl_cert = None
+            if config.fetch_ssl_certificate:
+                ssl_cert = SSLCertificate.from_url(url)
+
+            # Set up download handling
+            if self.browser_config.accept_downloads:
+                page.on(
+                    "download",
+                    lambda download: asyncio.create_task(
+                        self._handle_download(download)
+                    ),
+                )
+
+            # Handle page navigation and content loading
+            if not config.js_only:
+                await self.execute_hook("before_goto", page, context=context, url=url)
+
+                try:
+                    # Generate a unique nonce for this request
+                    nonce = hashlib.sha256(os.urandom(32)).hexdigest()
+                    
+                    # Add CSP headers to the request
+                    await page.set_extra_http_headers({
+                        'Content-Security-Policy': f"default-src 'self'; script-src 'self' 'nonce-{nonce}' 'strict-dynamic'"
+                    })
+
+                    response = await page.goto(
+                        url, wait_until=config.wait_until, timeout=config.page_timeout
+                    )
+                except Error as e:
+                    raise RuntimeError(f"Failed on navigating ACS-GOTO:\n{str(e)}")
+
+                await self.execute_hook("after_goto", page, context=context, url=url, response=response)
+
+                if response is None:
+                    status_code = 200
+                    response_headers = {}
+                else:
+                    status_code = response.status
+                    response_headers = response.headers
+
+            else:
+                status_code = 200
+                response_headers = {}
+
+            # Wait for body element and visibility
+            try:
+                await page.wait_for_selector("body", state="attached", timeout=30000)
+                
+                # Use the new check_visibility function with csp_compliant_wait
+                is_visible = await self.csp_compliant_wait(
+                    page,
+                    """() => {
+                        const element = document.body;
+                        if (!element) return false;
+                        const style = window.getComputedStyle(element);
+                        const isVisible = style.display !== 'none' && 
+                                        style.visibility !== 'hidden' && 
+                                        style.opacity !== '0';
+                        return isVisible;
+                    }""",
+                    timeout=30000
+                )
+                
+                if not is_visible and not config.ignore_body_visibility:
+                    visibility_info = await self.check_visibility(page)
+                    raise Error(f"Body element is hidden: {visibility_info}")
+
+            except Error as e:
+                visibility_info = await self.check_visibility(page)
+                
+                if self.config.verbose:
+                    self.logger.debug(
+                        message="Body visibility info: {info}",
+                        tag="DEBUG",
+                        params={"info": visibility_info},
+                    )
+
+                if not config.ignore_body_visibility:
+                    raise Error(f"Body element is hidden: {visibility_info}")            
+            
+            
+            # try:
+            #     await page.wait_for_selector("body", state="attached", timeout=30000)
+                
+            #     await page.wait_for_function(
+            #         """
+            #         () => {
+            #             const body = document.body;
+            #             const style = window.getComputedStyle(body);
+            #             return style.display !== 'none' && 
+            #                 style.visibility !== 'hidden' && 
+            #                 style.opacity !== '0';
+            #         }
+            #     """,
+            #         timeout=30000,
+            #     )
+            # except Error as e:
+            #     visibility_info = await page.evaluate(
+            #         """
+            #         () => {
+            #             const body = document.body;
+            #             const style = window.getComputedStyle(body);
+            #             return {
+            #                 display: style.display,
+            #                 visibility: style.visibility,
+            #                 opacity: style.opacity,
+            #                 hasContent: body.innerHTML.length,
+            #                 classList: Array.from(body.classList)
+            #             }
+            #         }
+            #     """
+            #     )
+
+            #     if self.config.verbose:
+            #         self.logger.debug(
+            #             message="Body visibility info: {info}",
+            #             tag="DEBUG",
+            #             params={"info": visibility_info},
+            #         )
+
+            #     if not config.ignore_body_visibility:
+            #         raise Error(f"Body element is hidden: {visibility_info}")
+
+            # Handle content loading and viewport adjustment
+            if not self.browser_config.text_mode and (
+                config.wait_for_images or config.adjust_viewport_to_content
+            ):
+                await page.wait_for_load_state("domcontentloaded")
+                await asyncio.sleep(0.1)
+                
+                # Check for image loading with improved error handling
+                images_loaded = await self.csp_compliant_wait(
+                    page,
+                    "() => Array.from(document.getElementsByTagName('img')).every(img => img.complete)",
+                    timeout=1000
+                )
+                
+                if not images_loaded and self.logger:
+                    self.logger.warning(
+                        message="Some images failed to load within timeout",
+                        tag="SCRAPE",
+                    )
+
+            # Adjust viewport if needed
+            if not self.browser_config.text_mode and config.adjust_viewport_to_content:
+                try:
+                    dimensions = await self.get_page_dimensions(page)
+                    page_height = dimensions['height']
+                    page_width = dimensions['width']                    
+                    # page_width = await page.evaluate(
+                    #     "document.documentElement.scrollWidth"
+                    # )
+                    # page_height = await page.evaluate(
+                    #     "document.documentElement.scrollHeight"
+                    # )
+
+                    target_width = self.browser_config.viewport_width
+                    target_height = int(target_width * page_width / page_height * 0.95)
+                    await page.set_viewport_size(
+                        {"width": target_width, "height": target_height}
+                    )
+
+                    scale = min(target_width / page_width, target_height / page_height)
+                    cdp = await page.context.new_cdp_session(page)
+                    await cdp.send(
+                        "Emulation.setDeviceMetricsOverride",
+                        {
+                            "width": page_width,
+                            "height": page_height,
+                            "deviceScaleFactor": 1,
+                            "mobile": False,
+                            "scale": scale,
+                        },
+                    )
+                except Exception as e:
+                    self.logger.warning(
+                        message="Failed to adjust viewport to content: {error}",
+                        tag="VIEWPORT",
+                        params={"error": str(e)},
+                    )
+
+            # Handle full page scanning
+            if config.scan_full_page:
+                await self._handle_full_page_scan(page, config.scroll_delay)
+
+            # Execute JavaScript if provided
+            # if config.js_code:
+            #     if isinstance(config.js_code, str):
+            #         await page.evaluate(config.js_code)
+            #     elif isinstance(config.js_code, list):
+            #         for js in config.js_code:
+            #             await page.evaluate(js)
+                        
+            if config.js_code:
+                # execution_result = await self.execute_user_script(page, config.js_code)
+                execution_result = await self.robust_execute_user_script(page, config.js_code)
+                if not execution_result["success"]:
+                    self.logger.warning(
+                        message="User script execution had issues: {error}",
+                        tag="JS_EXEC",
+                        params={"error": execution_result.get("error")}
+                    )                        
+
+                await self.execute_hook("on_execution_started", page, context=context)
+
+            # Handle user simulation
+            if config.simulate_user or config.magic:
+                await page.mouse.move(100, 100)
+                await page.mouse.down()
+                await page.mouse.up()
+                await page.keyboard.press("ArrowDown")
+
+            # Handle wait_for condition
+            if config.wait_for:
+                try:
+                    await self.smart_wait(
+                        page, config.wait_for, timeout=config.page_timeout
+                    )
+                except Exception as e:
+                    raise RuntimeError(f"Wait condition failed: {str(e)}")
+
+            # Update image dimensions if needed
+            if not self.browser_config.text_mode:
+                update_image_dimensions_js = load_js_script("update_image_dimensions")
+                try:
+                    try:
+                        await page.wait_for_load_state("domcontentloaded", timeout=5)
+                    except PlaywrightTimeoutError:
+                        pass
+                    await page.evaluate(update_image_dimensions_js)
+                except Exception as e:
+                    self.logger.error(
+                        message="Error updating image dimensions: {error}",
+                        tag="ERROR",
+                        params={"error": str(e)},
+                    )
+
+            # Process iframes if needed
+            if config.process_iframes:
+                page = await self.process_iframes(page)
+
+            # Pre-content retrieval hooks and delay
+            await self.execute_hook("before_retrieve_html", page, context=context)
+            if config.delay_before_return_html:
+                await asyncio.sleep(config.delay_before_return_html)
+
+            # Handle overlay removal
+            if config.remove_overlay_elements:
+                await self.remove_overlay_elements(page)
+
+            # Get final HTML content
+            html = await page.content()
+            await self.execute_hook("before_return_html", page = page, html = html, context=context)
+
+            # Handle PDF and screenshot generation
+            start_export_time = time.perf_counter()
+            pdf_data = None
+            screenshot_data = None
+
+            if config.pdf:
+                pdf_data = await self.export_pdf(page)
+
+            if config.screenshot:
+                if config.screenshot_wait_for:
+                    await asyncio.sleep(config.screenshot_wait_for)
+                screenshot_data = await self.take_screenshot(
+                    page, screenshot_height_threshold=config.screenshot_height_threshold
+                )
+
+            if screenshot_data or pdf_data:
+                self.logger.info(
+                    message="Exporting PDF and taking screenshot took {duration:.2f}s",
+                    tag="EXPORT",
+                    params={"duration": time.perf_counter() - start_export_time},
+                )
+
+            # Define delayed content getter
+            async def get_delayed_content(delay: float = 5.0) -> str:
+                self.logger.info(
+                    message="Waiting for {delay} seconds before retrieving content for {url}",
+                    tag="INFO",
+                    params={"delay": delay, "url": url},
+                )
+                await asyncio.sleep(delay)
+                return await page.content()
+
+            # Return complete response
+            return AsyncCrawlResponse(
+                html=html,
+                response_headers=response_headers,
+                status_code=status_code,
+                screenshot=screenshot_data,
+                pdf_data=pdf_data,
+                get_delayed_content=get_delayed_content,
+                ssl_certificate=ssl_cert,
+                downloaded_files=(
+                    self._downloaded_files if self._downloaded_files else None
+                ),
+            )
+
+        except Exception as e:
+            raise e
+        
+        finally:
+            # If no session_id is given we should close the page
+            if not config.session_id:
+                await page.close()
+
+    async def _handle_full_page_scan(self, page: Page, scroll_delay: float = 0.1):
+        """
+        Helper method to handle full page scanning. 
+        
+        How it works:
+        1. Get the viewport height.
+        2. Scroll to the bottom of the page.
+        3. Get the total height of the page.
+        4. Scroll back to the top of the page.
+        5. Scroll to the bottom of the page again.  
+        6. Continue scrolling until the bottom of the page is reached.
+        
+        Args:
+            page (Page): The Playwright page object
+            scroll_delay (float): The delay between page scrolls
+        
+        """
+        try:
+            viewport_height = page.viewport_size.get(
+                "height", self.browser_config.viewport_height
+            )
+            current_position = viewport_height
+
+            # await page.evaluate(f"window.scrollTo(0, {current_position})")
+            await self.safe_scroll(page, 0, current_position, delay=scroll_delay)
+            # await self.csp_scroll_to(page, 0, current_position)
+            # await asyncio.sleep(scroll_delay)
+
+            # total_height = await page.evaluate("document.documentElement.scrollHeight")
+            dimensions = await self.get_page_dimensions(page)
+            total_height = dimensions['height']
+            
+            while current_position < total_height:
+                current_position = min(current_position + viewport_height, total_height)
+                await self.safe_scroll(page, 0, current_position, delay=scroll_delay)
+                # await page.evaluate(f"window.scrollTo(0, {current_position})")
+                # await asyncio.sleep(scroll_delay)
+
+                # new_height = await page.evaluate("document.documentElement.scrollHeight")
+                dimensions = await self.get_page_dimensions(page)
+                new_height = dimensions['height']
+                
+                if new_height > total_height:
+                    total_height = new_height
+
+            # await page.evaluate("window.scrollTo(0, 0)")
+            await self.safe_scroll(page, 0, 0)
+
+        except Exception as e:
+            self.logger.warning(
+                message="Failed to perform full page scan: {error}",
+                tag="PAGE_SCAN",
+                params={"error": str(e)},
+            )
+        else:
+            # await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+            await self.safe_scroll(page, 0, total_height)
+
+    async def _handle_download(self, download):
+        """
+        Handle file downloads.
+        
+        How it works:
+        1. Get the suggested filename.
+        2. Get the download path.
+        3. Log the download.
+        4. Start the download.
+        5. Save the downloaded file.
+        6. Log the completion.
+        
+        Args:
+            download (Download): The Playwright download object
+            
+        Returns:
+            None
+        """
+        try:
+            suggested_filename = download.suggested_filename
+            download_path = os.path.join(self.downloads_path, suggested_filename)
+
+            self.logger.info(
+                message="Downloading {filename} to {path}",
+                tag="FETCH",
+                params={"filename": suggested_filename, "path": download_path},
+            )
+
+            start_time = time.perf_counter()
+            await download.save_as(download_path)
+            end_time = time.perf_counter()
+            self._downloaded_files.append(download_path)
+
+            self.logger.success(
+                message="Downloaded {filename} successfully",
+                tag="COMPLETE",
+                params={
+                    "filename": suggested_filename,
+                    "path": download_path,
+                    "duration": f"{end_time - start_time:.2f}s",
+                },
+            )
+        except Exception as e:
+            self.logger.error(
+                message="Failed to handle download: {error}",
+                tag="ERROR",
+                params={"error": str(e)},
+            )
+
+    async def remove_overlay_elements(self, page: Page) -> None:
+        """
+        Removes popup overlays, modals, cookie notices, and other intrusive elements from the page.
+
+        Args:
+            page (Page): The Playwright page instance
+        """
+        remove_overlays_js = load_js_script("remove_overlay_elements")
+
+        try:
+            await page.evaluate(f"""
+                (() => {{
+                    try {{
+                        {remove_overlays_js}
+                        return {{ success: true }};
+                    }} catch (error) {{
+                        return {{
+                            success: false,
+                            error: error.toString(),
+                            stack: error.stack
+                        }};
+                    }}
+                }})()
+            """)
+            await page.wait_for_timeout(500)  # Wait for any animations to complete
+        except Exception as e:
+            self.logger.warning(
+                message="Failed to remove overlay elements: {error}",
+                tag="SCRAPE",
+                params={"error": str(e)},
+            )
+
+    async def export_pdf(self, page: Page) -> bytes:
+        """
+        Exports the current page as a PDF.
+        
+        Args:
+            page (Page): The Playwright page object
+            
+        Returns:
+            bytes: The PDF data
+        """
+        pdf_data = await page.pdf(print_background=True)
+        return pdf_data
+
+    async def take_screenshot(self, page, **kwargs) -> str:
+        """
+        Take a screenshot of the current page.
+        
+        Args:
+            page (Page): The Playwright page object
+            kwargs: Additional keyword arguments
+        
+        Returns:
+            str: The base64-encoded screenshot data
+        """
+        need_scroll = await self.page_need_scroll(page)
+        
+        if not need_scroll:
+            # Page is short enough, just take a screenshot
+            return await self.take_screenshot_naive(page)
+        else:
+            # Page is too long, try to take a full-page screenshot
+            return await self.take_screenshot_scroller(page, **kwargs)
+            # return await self.take_screenshot_from_pdf(await self.export_pdf(page))
+
+    async def take_screenshot_from_pdf(self, pdf_data: bytes) -> str:
+        """
+        Convert the first page of the PDF to a screenshot.     
+        
+        Requires pdf2image and poppler.
+        
+        Args:
+            pdf_data (bytes): The PDF data
+        
+        Returns:
+            str: The base64-encoded screenshot data
+        """
+        try:
+            from pdf2image import convert_from_bytes
+
+            images = convert_from_bytes(pdf_data)
+            final_img = images[0].convert("RGB")
+            buffered = BytesIO()
+            final_img.save(buffered, format="JPEG")
+            return base64.b64encode(buffered.getvalue()).decode("utf-8")
+        except Exception as e:
+            error_message = f"Failed to take PDF-based screenshot: {str(e)}"
+            self.logger.error(
+                message="PDF Screenshot failed: {error}",
+                tag="ERROR",
+                params={"error": error_message},
+            )
+            # Return error image as fallback
+            img = Image.new("RGB", (800, 600), color="black")
+            draw = ImageDraw.Draw(img)
+            font = ImageFont.load_default()
+            draw.text((10, 10), error_message, fill=(255, 255, 255), font=font)
+            buffered = BytesIO()
+            img.save(buffered, format="JPEG")
+            return base64.b64encode(buffered.getvalue()).decode("utf-8")
+
+    async def take_screenshot_scroller(self, page: Page, **kwargs) -> str:
+        """
+        Attempt to set a large viewport and take a full-page screenshot.
+        If still too large, segment the page as before.
+        
+        Requires pdf2image and poppler.
+        
+        Args:
+            page (Page): The Playwright page object
+            kwargs: Additional keyword arguments
+            
+        Returns:
+            str: The base64-encoded screenshot data
+        """
+        try:
+            # Get page height
+            dimensions = await self.get_page_dimensions(page)
+            page_width = dimensions['width']
+            page_height = dimensions['height']            
+            # page_height = await page.evaluate("document.documentElement.scrollHeight")
+            # page_width = await page.evaluate("document.documentElement.scrollWidth")
+
+            # Set a large viewport
+            large_viewport_height = min(
+                page_height,
+                kwargs.get("screenshot_height_threshold", SCREENSHOT_HEIGHT_TRESHOLD),
+            )
+            await page.set_viewport_size(
+                {"width": page_width, "height": large_viewport_height}
+            )
+
+            # Page still too long, segment approach
+            segments = []
+            viewport_size = page.viewport_size
+            viewport_height = viewport_size["height"]
+
+            num_segments = (page_height // viewport_height) + 1
+            for i in range(num_segments):
+                y_offset = i * viewport_height
+                await page.evaluate(f"window.scrollTo(0, {y_offset})")
+                await asyncio.sleep(0.01)  # wait for render
+                seg_shot = await page.screenshot(full_page=False)
+                img = Image.open(BytesIO(seg_shot)).convert("RGB")
+                segments.append(img)
+
+            total_height = sum(img.height for img in segments)
+            stitched = Image.new("RGB", (segments[0].width, total_height))
+            offset = 0
+            for img in segments:
+                # stitched.paste(img, (0, offset))
+                stitched.paste(img.convert("RGB"), (0, offset))
+                offset += img.height
+
+            buffered = BytesIO()
+            stitched = stitched.convert("RGB")
+            stitched.save(buffered, format="BMP", quality=85)
+            encoded = base64.b64encode(buffered.getvalue()).decode("utf-8")
+
+            return encoded
+        except Exception as e:
+            error_message = f"Failed to take large viewport screenshot: {str(e)}"
+            self.logger.error(
+                message="Large viewport screenshot failed: {error}",
+                tag="ERROR",
+                params={"error": error_message},
+            )
+            # return error image
+            img = Image.new("RGB", (800, 600), color="black")
+            draw = ImageDraw.Draw(img)
+            font = ImageFont.load_default()
+            draw.text((10, 10), error_message, fill=(255, 255, 255), font=font)
+            buffered = BytesIO()
+            img.save(buffered, format="JPEG")
+            return base64.b64encode(buffered.getvalue()).decode("utf-8")
+        finally:
+            await page.close()
+
+    async def take_screenshot_naive(self, page: Page) -> str:
+        """
+        Takes a screenshot of the current page.
+
+        Args:
+            page (Page): The Playwright page instance
+
+        Returns:
+            str: Base64-encoded screenshot image
+        """
+        try:
+            # The page is already loaded, just take the screenshot
+            screenshot = await page.screenshot(full_page=False)
+            return base64.b64encode(screenshot).decode("utf-8")
+        except Exception as e:
+            error_message = f"Failed to take screenshot: {str(e)}"
+            self.logger.error(
+                message="Screenshot failed: {error}",
+                tag="ERROR",
+                params={"error": error_message},
+            )
+
+            # Generate an error image
+            img = Image.new("RGB", (800, 600), color="black")
+            draw = ImageDraw.Draw(img)
+            font = ImageFont.load_default()
+            draw.text((10, 10), error_message, fill=(255, 255, 255), font=font)
+
+            buffered = BytesIO()
+            img.save(buffered, format="JPEG")
+            return base64.b64encode(buffered.getvalue()).decode("utf-8")
+        finally:
+            await page.close()
+
+    async def export_storage_state(self, path: str = None) -> dict:
+        """
+        Exports the current storage state (cookies, localStorage, sessionStorage)
+        to a JSON file at the specified path.
+        
+        Args:
+            path (str): The path to save the storage state JSON file
+        
+        Returns:
+            dict: The exported storage state
+        """
+        if self.default_context:
+            state = await self.default_context.storage_state(path=path)
+            self.logger.info(
+                message="Exported storage state to {path}",
+                tag="INFO",
+                params={"path": path},
+            )
+            return state
+        else:
+            self.logger.warning(
+                message="No default_context available to export storage state.",
+                tag="WARNING",
+            )
+
+    async def robust_execute_user_script(self, page: Page, js_code: Union[str, List[str]]) -> Dict[str, Any]:
+        """
+        Executes user-provided JavaScript code with proper error handling and context,
+        supporting both synchronous and async user code, plus navigations.
+        
+        How it works:
+        1. Wait for load state 'domcontentloaded'
+        2. If js_code is a string, execute it directly
+        3. If js_code is a list, execute each element in sequence
+        4. Wait for load state 'networkidle'        
+        5. Return results   
+        
+        Args:    
+            page (Page): The Playwright page instance
+            js_code (Union[str, List[str]]): The JavaScript code to execute
+        
+        Returns:
+            Dict[str, Any]: The results of the execution
+        """
+        try:
+            await page.wait_for_load_state('domcontentloaded')
+            
+            if isinstance(js_code, str):
+                scripts = [js_code]
+            else:
+                scripts = js_code
+            
+            results = []
+            for script in scripts:
+                try:
+                    # Attempt the evaluate
+                    # If the user code triggers navigation, we catch the "context destroyed" error
+                    # then wait for the new page to load before continuing
+                    result = None
+                    try:
+                        result = await page.evaluate(f"""
+                        (async () => {{
+                            try {{
+                                {script}
+                                return {{ success: true }};
+                            }} catch (err) {{
+                                return {{ success: false, error: err.toString(), stack: err.stack }};
+                            }}
+                        }})();
+                        """)
+                    except Error as e:
+                        # If it's due to navigation destroying the context, handle gracefully
+                        if "Execution context was destroyed" in str(e):
+                            self.logger.info("Navigation triggered by script, waiting for load state", tag="JS_EXEC")
+                            try:
+                                await page.wait_for_load_state('load', timeout=30000)
+                            except Error as nav_err:
+                                self.logger.warning(
+                                    message="Navigation wait failed: {error}",
+                                    tag="JS_EXEC",
+                                    params={"error": str(nav_err)}
+                                )
+                            try:
+                                await page.wait_for_load_state('networkidle', timeout=30000)
+                            except Error as nav_err:
+                                self.logger.warning(
+                                    message="Network idle wait failed: {error}",
+                                    tag="JS_EXEC",
+                                    params={"error": str(nav_err)}
+                                )
+                            # Return partial success, or adapt as you see fit
+                            result = {
+                                "success": True,
+                                "info": "Navigation triggered, ignoring context destroyed error"
+                            }
+                        else:
+                            # It's some other error, log and continue
+                            self.logger.error(
+                                message="Playwright execution error: {error}",
+                                tag="JS_EXEC",
+                                params={"error": str(e)}
+                            )
+                            result = {"success": False, "error": str(e)}
+                    
+                    # If we made it this far with no repeated error, do post-load waits
+                    t1 = time.time()
+                    try:
+                        await page.wait_for_load_state('domcontentloaded', timeout=5000)
+                        print("DOM content loaded after script execution in", time.time() - t1)
+                    except Error as e:
+                        self.logger.warning(
+                            message="DOM content load timeout: {error}",
+                            tag="JS_EXEC",
+                            params={"error": str(e)}
+                        )
+                    
+                    # t1 = time.time()
+                    # try:
+                    #     await page.wait_for_load_state('networkidle', timeout=5000)
+                    #     print("Network idle after script execution in", time.time() - t1)
+                    # except Error as e:
+                    #     self.logger.warning(
+                    #         message="Network idle timeout: {error}",
+                    #         tag="JS_EXEC",
+                    #         params={"error": str(e)}
+                    #     )
+
+                    results.append(result if result else {"success": True})
+
+                except Exception as e:
+                    # Catch anything else
+                    self.logger.error(
+                        message="Script chunk failed: {error}",
+                        tag="JS_EXEC",
+                        params={"error": str(e)}
+                    )
+                    results.append({"success": False, "error": str(e)})
+
+            return {"success": True, "results": results}
+        
+        except Exception as e:
+            self.logger.error(
+                message="Script execution failed: {error}",
+                tag="JS_EXEC",
+                params={"error": str(e)}
+            )
+            return {"success": False, "error": str(e)}
+
+    async def execute_user_script(self, page: Page, js_code: Union[str, List[str]]) -> Dict[str, Any]:
+        """
+        Executes user-provided JavaScript code with proper error handling and context.
+        
+        Args:
+            page: Playwright page object
+            js_code: Single JavaScript string or list of JavaScript code strings
+            
+        Returns:
+            Dict containing execution status and results/errors
+        """
+        try:
+            # Ensure the page is ready for script execution
+            await page.wait_for_load_state('domcontentloaded')
+            
+            # Handle single script or multiple scripts
+            if isinstance(js_code, str):
+                scripts = [js_code]
+            else:
+                scripts = js_code
+                
+            results = []
+            for script in scripts:
+                try:
+                    # Execute the script and wait for network idle
+                    result = await page.evaluate(f"""
+                        (() => {{
+                            return new Promise((resolve) => {{
+                                try {{
+                                    const result = (function() {{
+                                        {script}
+                                    }})();
+                                    
+                                    // If result is a promise, wait for it
+                                    if (result instanceof Promise) {{
+                                        result.then(() => {{
+                                            // Wait a bit for any triggered effects
+                                            setTimeout(() => resolve({{ success: true }}), 100);
+                                        }}).catch(error => {{
+                                            resolve({{
+                                                success: false,
+                                                error: error.toString(),
+                                                stack: error.stack
+                                            }});
+                                        }});
+                                    }} else {{
+                                        // For non-promise results, still wait a bit for effects
+                                        setTimeout(() => resolve({{ success: true }}), 100);
+                                    }}
+                                }} catch (error) {{
+                                    resolve({{
+                                        success: false,
+                                        error: error.toString(),
+                                        stack: error.stack
+                                    }});
+                                }}
+                            }});
+                        }})()
+                    """)
+                    
+                    # Wait for network idle after script execution
+                    t1 = time.time()
+                    await page.wait_for_load_state('domcontentloaded', timeout=5000)
+                    print("DOM content loaded after script execution in", time.time() - t1)
+
+                    t1 = time.time()
+                    await page.wait_for_load_state('networkidle', timeout=5000)
+                    print("Network idle after script execution in", time.time() - t1)
+                    
+                    results.append(result if result else {"success": True})
+                    
+                except Error as e:
+                    # Handle Playwright-specific errors
+                    self.logger.error(
+                        message="Playwright execution error: {error}",
+                        tag="JS_EXEC",
+                        params={"error": str(e)}
+                    )
+                    results.append({"success": False, "error": str(e)})
+                    
+            return {"success": True, "results": results}
+            
+        except Exception as e:
+            self.logger.error(
+                message="Script execution failed: {error}",
+                tag="JS_EXEC",
+                params={"error": str(e)}
+            )
+            return {"success": False, "error": str(e)}
+            
+        except Exception as e:
+            self.logger.error(
+                message="Script execution failed: {error}",
+                tag="JS_EXEC",
+                params={"error": str(e)}
+            )
+            return {"success": False, "error": str(e)}
+
+    async def check_visibility(self, page):
+        """
+        Checks if an element is visible on the page.
+        
+        Args:
+            page: Playwright page object
+            
+        Returns:
+            Boolean indicating visibility
+        """
+        return await page.evaluate("""
+            () => {
+                const element = document.body;
+                if (!element) return false;
+                const style = window.getComputedStyle(element);
+                const isVisible = style.display !== 'none' && 
+                                style.visibility !== 'hidden' && 
+                                style.opacity !== '0';
+                return isVisible;
+            }
+        """)       
+        
+    async def safe_scroll(self, page: Page, x: int, y: int, delay: float = 0.1):
+        """
+        Safely scroll the page with rendering time.
+        
+        Args:
+            page: Playwright page object
+            x: Horizontal scroll position
+            y: Vertical scroll position
+        """
+        result = await self.csp_scroll_to(page, x, y)
+        if result['success']:
+            await page.wait_for_timeout(delay * 1000)
+        return result
+            
+    async def csp_scroll_to(self, page: Page, x: int, y: int) -> Dict[str, Any]:
+        """
+        Performs a CSP-compliant scroll operation and returns the result status.
+        
+        Args:
+            page: Playwright page object
+            x: Horizontal scroll position
+            y: Vertical scroll position
+            
+        Returns:
+            Dict containing scroll status and position information
+        """
+        try:
+            result = await page.evaluate(
+                f"""() => {{
+                    try {{
+                        const startX = window.scrollX;
+                        const startY = window.scrollY;
+                        window.scrollTo({x}, {y});
+                        
+                        // Get final position after scroll
+                        const endX = window.scrollX;
+                        const endY = window.scrollY;
+                        
+                        return {{
+                            success: true,
+                            startPosition: {{ x: startX, y: startY }},
+                            endPosition: {{ x: endX, y: endY }},
+                            targetPosition: {{ x: {x}, y: {y} }},
+                            delta: {{
+                                x: Math.abs(endX - {x}),
+                                y: Math.abs(endY - {y})
+                            }}
+                        }};
+                    }} catch (e) {{
+                        return {{
+                            success: false,
+                            error: e.toString()
+                        }};
+                    }}
+                }}"""
+            )
+            
+            if not result['success']:
+                self.logger.warning(
+                    message="Scroll operation failed: {error}",
+                    tag="SCROLL",
+                    params={"error": result.get('error')}
+                )
+                
+            return result
+            
+        except Exception as e:
+            self.logger.error(
+                message="Failed to execute scroll: {error}",
+                tag="SCROLL",
+                params={"error": str(e)}
+            )
+            return {
+                "success": False,
+                "error": str(e)
+            }
+        
+    async def get_page_dimensions(self, page: Page):
+        """
+        Get the dimensions of the page.
+        
+        Args:
+            page: Playwright page object
+            
+        Returns:
+            Dict containing width and height of the page
+        """
+        return await page.evaluate("""
+            () => {
+                const {scrollWidth, scrollHeight} = document.documentElement;
+                return {width: scrollWidth, height: scrollHeight};
+            }
+        """)
+    
+    async def page_need_scroll(self, page: Page) -> bool:
+        """
+        Determine whether the page need to scroll
+        
+        Args:
+            page: Playwright page object
+            
+        Returns:
+            bool: True if page needs scrolling
+        """
+        try:
+            need_scroll = await page.evaluate("""
+            () => {
+                const scrollHeight = document.documentElement.scrollHeight;
+                const viewportHeight = window.innerHeight;
+                return scrollHeight > viewportHeight;
+            }
+            """)
+            return need_scroll
+        except Exception as e:
+            self.logger.warning(
+                message="Failed to check scroll need: {error}. Defaulting to True for safety.",
+                tag="SCROLL",
+                params={"error": str(e)}
+            )
+            return True  # Default to scrolling if check fails

crawl4ai/async_database.py

@@ -0,0 +1,495 @@
+import os, sys
+from pathlib import Path
+import aiosqlite
+import asyncio
+from typing import Optional, Tuple, Dict
+from contextlib import asynccontextmanager
+import logging
+import json  # Added for serialization/deserialization
+from .utils import ensure_content_dirs, generate_content_hash
+from .models import CrawlResult, MarkdownGenerationResult
+import xxhash
+import aiofiles
+from .config import NEED_MIGRATION
+from .version_manager import VersionManager
+from .async_logger import AsyncLogger
+from .utils import get_error_context, create_box_message
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+base_directory = DB_PATH = os.path.join(os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai")
+os.makedirs(DB_PATH, exist_ok=True)
+DB_PATH = os.path.join(base_directory, "crawl4ai.db")
+
+class AsyncDatabaseManager:
+    def __init__(self, pool_size: int = 10, max_retries: int = 3):
+        self.db_path = DB_PATH
+        self.content_paths = ensure_content_dirs(os.path.dirname(DB_PATH))
+        self.pool_size = pool_size
+        self.max_retries = max_retries
+        self.connection_pool: Dict[int, aiosqlite.Connection] = {}
+        self.pool_lock = asyncio.Lock()
+        self.init_lock = asyncio.Lock()
+        self.connection_semaphore = asyncio.Semaphore(pool_size)
+        self._initialized = False  
+        self.version_manager = VersionManager()
+        self.logger = AsyncLogger(
+            log_file=os.path.join(base_directory, ".crawl4ai", "crawler_db.log"),
+            verbose=False,
+            tag_width=10
+        )
+        
+        
+    async def initialize(self):
+        """Initialize the database and connection pool"""
+        try:
+            self.logger.info("Initializing database", tag="INIT")
+            # Ensure the database file exists
+            os.makedirs(os.path.dirname(self.db_path), exist_ok=True)
+            
+            # Check if version update is needed
+            needs_update = self.version_manager.needs_update()
+            
+            # Always ensure base table exists
+            await self.ainit_db()
+            
+            # Verify the table exists
+            async with aiosqlite.connect(self.db_path, timeout=30.0) as db:
+                async with db.execute(
+                    "SELECT name FROM sqlite_master WHERE type='table' AND name='crawled_data'"
+                ) as cursor:
+                    result = await cursor.fetchone()
+                    if not result:
+                        raise Exception("crawled_data table was not created")
+            
+            # If version changed or fresh install, run updates
+            if needs_update:
+                self.logger.info("New version detected, running updates", tag="INIT")
+                await self.update_db_schema()
+                from .migrations import run_migration  # Import here to avoid circular imports
+                await run_migration()
+                self.version_manager.update_version()  # Update stored version after successful migration
+                self.logger.success("Version update completed successfully", tag="COMPLETE")
+            else:
+                self.logger.success("Database initialization completed successfully", tag="COMPLETE")
+
+                
+        except Exception as e:
+            self.logger.error(
+                message="Database initialization error: {error}",
+                tag="ERROR",
+                params={"error": str(e)}
+            )
+            self.logger.info(
+                message="Database will be initialized on first use",
+                tag="INIT"
+            )
+                        
+            raise
+
+            
+    async def cleanup(self):
+        """Cleanup connections when shutting down"""
+        async with self.pool_lock:
+            for conn in self.connection_pool.values():
+                await conn.close()
+            self.connection_pool.clear()
+
+    @asynccontextmanager
+    async def get_connection(self):
+        """Connection pool manager with enhanced error handling"""
+        if not self._initialized:
+            async with self.init_lock:
+                if not self._initialized:
+                    try:
+                        await self.initialize()
+                        self._initialized = True
+                    except Exception as e:
+                        import sys
+                        error_context = get_error_context(sys.exc_info())
+                        self.logger.error(
+                            message="Database initialization failed:\n{error}\n\nContext:\n{context}\n\nTraceback:\n{traceback}",
+                            tag="ERROR",
+                            force_verbose=True,
+                            params={
+                                "error": str(e),
+                                "context": error_context["code_context"],
+                                "traceback": error_context["full_traceback"]
+                            }
+                        )
+                        raise
+
+        await self.connection_semaphore.acquire()
+        task_id = id(asyncio.current_task())
+        
+        try:
+            async with self.pool_lock:
+                if task_id not in self.connection_pool:
+                    try:
+                        conn = await aiosqlite.connect(
+                            self.db_path,
+                            timeout=30.0
+                        )
+                        await conn.execute('PRAGMA journal_mode = WAL')
+                        await conn.execute('PRAGMA busy_timeout = 5000')
+                        
+                        # Verify database structure
+                        async with conn.execute("PRAGMA table_info(crawled_data)") as cursor:
+                            columns = await cursor.fetchall()
+                            column_names = [col[1] for col in columns]
+                            expected_columns = {
+                                'url', 'html', 'cleaned_html', 'markdown', 'extracted_content',
+                                'success', 'media', 'links', 'metadata', 'screenshot',
+                                'response_headers', 'downloaded_files'
+                            }
+                            missing_columns = expected_columns - set(column_names)
+                            if missing_columns:
+                                raise ValueError(f"Database missing columns: {missing_columns}")
+                        
+                        self.connection_pool[task_id] = conn
+                    except Exception as e:
+                        import sys
+                        error_context = get_error_context(sys.exc_info())
+                        error_message = (
+                            f"Unexpected error in db get_connection at line {error_context['line_no']} "
+                            f"in {error_context['function']} ({error_context['filename']}):\n"
+                            f"Error: {str(e)}\n\n"
+                            f"Code context:\n{error_context['code_context']}"
+                        )
+                        self.logger.error(
+                            message=create_box_message(error_message, type= "error"),
+                        )
+
+                        raise
+
+            yield self.connection_pool[task_id]
+
+        except Exception as e:
+            import sys
+            error_context = get_error_context(sys.exc_info())
+            error_message = (
+                f"Unexpected error in db get_connection at line {error_context['line_no']} "
+                f"in {error_context['function']} ({error_context['filename']}):\n"
+                f"Error: {str(e)}\n\n"
+                f"Code context:\n{error_context['code_context']}"
+            )
+            self.logger.error(
+                message=create_box_message(error_message, type= "error"),
+            )
+            raise
+        finally:
+            async with self.pool_lock:
+                if task_id in self.connection_pool:
+                    await self.connection_pool[task_id].close()
+                    del self.connection_pool[task_id]
+            self.connection_semaphore.release()
+
+
+    async def execute_with_retry(self, operation, *args):
+        """Execute database operations with retry logic"""
+        for attempt in range(self.max_retries):
+            try:
+                async with self.get_connection() as db:
+                    result = await operation(db, *args)
+                    await db.commit()
+                    return result
+            except Exception as e:
+                if attempt == self.max_retries - 1:
+                    self.logger.error(
+                        message="Operation failed after {retries} attempts: {error}",
+                        tag="ERROR",
+                        force_verbose=True,
+                        params={
+                            "retries": self.max_retries,
+                            "error": str(e)
+                        }
+                    )                    
+                    raise
+                await asyncio.sleep(1 * (attempt + 1))  # Exponential backoff
+
+    async def ainit_db(self):
+        """Initialize database schema"""
+        async with aiosqlite.connect(self.db_path, timeout=30.0) as db:
+            await db.execute('''
+                CREATE TABLE IF NOT EXISTS crawled_data (
+                    url TEXT PRIMARY KEY,
+                    html TEXT,
+                    cleaned_html TEXT,
+                    markdown TEXT,
+                    extracted_content TEXT,
+                    success BOOLEAN,
+                    media TEXT DEFAULT "{}",
+                    links TEXT DEFAULT "{}",
+                    metadata TEXT DEFAULT "{}",
+                    screenshot TEXT DEFAULT "",
+                    response_headers TEXT DEFAULT "{}",
+                    downloaded_files TEXT DEFAULT "{}"  -- New column added
+                )
+            ''')
+            await db.commit()
+
+        
+
+    async def update_db_schema(self):
+        """Update database schema if needed"""
+        async with aiosqlite.connect(self.db_path, timeout=30.0) as db:
+            cursor = await db.execute("PRAGMA table_info(crawled_data)")
+            columns = await cursor.fetchall()
+            column_names = [column[1] for column in columns]
+            
+            # List of new columns to add
+            new_columns = ['media', 'links', 'metadata', 'screenshot', 'response_headers', 'downloaded_files']
+            
+            for column in new_columns:
+                if column not in column_names:
+                    await self.aalter_db_add_column(column, db)
+            await db.commit()
+
+    async def aalter_db_add_column(self, new_column: str, db):
+        """Add new column to the database"""
+        if new_column == 'response_headers':
+            await db.execute(f'ALTER TABLE crawled_data ADD COLUMN {new_column} TEXT DEFAULT "{{}}"')
+        else:
+            await db.execute(f'ALTER TABLE crawled_data ADD COLUMN {new_column} TEXT DEFAULT ""')
+        self.logger.info(
+            message="Added column '{column}' to the database",
+            tag="INIT",
+            params={"column": new_column}
+        )        
+
+
+    async def aget_cached_url(self, url: str) -> Optional[CrawlResult]:
+        """Retrieve cached URL data as CrawlResult"""
+        async def _get(db):
+            async with db.execute(
+                'SELECT * FROM crawled_data WHERE url = ?', (url,)
+            ) as cursor:
+                row = await cursor.fetchone()
+                if not row:
+                    return None
+                    
+                # Get column names
+                columns = [description[0] for description in cursor.description]
+                # Create dict from row data
+                row_dict = dict(zip(columns, row))
+                
+                # Load content from files using stored hashes
+                content_fields = {
+                    'html': row_dict['html'],
+                    'cleaned_html': row_dict['cleaned_html'],
+                    'markdown': row_dict['markdown'],
+                    'extracted_content': row_dict['extracted_content'],
+                    'screenshot': row_dict['screenshot'],
+                    'screenshots': row_dict['screenshot'],
+                }
+                
+                for field, hash_value in content_fields.items():
+                    if hash_value:
+                        content = await self._load_content(
+                            hash_value, 
+                            field.split('_')[0]  # Get content type from field name
+                        )
+                        row_dict[field] = content or ""
+                    else:
+                        row_dict[field] = ""
+
+                # Parse JSON fields
+                json_fields = ['media', 'links', 'metadata', 'response_headers', 'markdown']
+                for field in json_fields:
+                    try:
+                        row_dict[field] = json.loads(row_dict[field]) if row_dict[field] else {}
+                    except json.JSONDecodeError:
+                        row_dict[field] = {}
+
+                if isinstance(row_dict['markdown'], Dict):
+                    row_dict['markdown_v2'] = row_dict['markdown']
+                    if row_dict['markdown'].get('raw_markdown'):
+                        row_dict['markdown'] = row_dict['markdown']['raw_markdown']
+                
+                # Parse downloaded_files
+                try:
+                    row_dict['downloaded_files'] = json.loads(row_dict['downloaded_files']) if row_dict['downloaded_files'] else []
+                except json.JSONDecodeError:
+                    row_dict['downloaded_files'] = []
+
+                # Remove any fields not in CrawlResult model
+                valid_fields = CrawlResult.__annotations__.keys()
+                filtered_dict = {k: v for k, v in row_dict.items() if k in valid_fields}
+                
+                return CrawlResult(**filtered_dict)
+
+        try:
+            return await self.execute_with_retry(_get)
+        except Exception as e:
+            self.logger.error(
+                message="Error retrieving cached URL: {error}",
+                tag="ERROR",
+                force_verbose=True,
+                params={"error": str(e)}
+            )
+            return None
+
+    async def acache_url(self, result: CrawlResult):
+        """Cache CrawlResult data"""
+        # Store content files and get hashes
+        content_map = {
+            'html': (result.html, 'html'),
+            'cleaned_html': (result.cleaned_html or "", 'cleaned'),
+            'markdown': None,
+            'extracted_content': (result.extracted_content or "", 'extracted'),
+            'screenshot': (result.screenshot or "", 'screenshots')
+        }
+
+        try:
+            if isinstance(result.markdown, MarkdownGenerationResult):
+                content_map['markdown'] = (result.markdown.model_dump_json(), 'markdown')
+            elif hasattr(result, 'markdown_v2'):
+                content_map['markdown'] = (result.markdown_v2.model_dump_json(), 'markdown')
+            elif isinstance(result.markdown, str):
+                markdown_result = MarkdownGenerationResult(raw_markdown=result.markdown)
+                content_map['markdown'] = (markdown_result.model_dump_json(), 'markdown')
+            else:
+                content_map['markdown'] = (MarkdownGenerationResult().model_dump_json(), 'markdown')
+        except Exception as e:
+            self.logger.warning(
+                message=f"Error processing markdown content: {str(e)}",
+                tag="WARNING"
+            )
+            # Fallback to empty markdown result
+            content_map['markdown'] = (MarkdownGenerationResult().model_dump_json(), 'markdown')
+        
+        content_hashes = {}
+        for field, (content, content_type) in content_map.items():
+            content_hashes[field] = await self._store_content(content, content_type)
+
+        async def _cache(db):
+            await db.execute('''
+                INSERT INTO crawled_data (
+                    url, html, cleaned_html, markdown,
+                    extracted_content, success, media, links, metadata,
+                    screenshot, response_headers, downloaded_files
+                )
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(url) DO UPDATE SET
+                    html = excluded.html,
+                    cleaned_html = excluded.cleaned_html,
+                    markdown = excluded.markdown,
+                    extracted_content = excluded.extracted_content,
+                    success = excluded.success,
+                    media = excluded.media,
+                    links = excluded.links,
+                    metadata = excluded.metadata,
+                    screenshot = excluded.screenshot,
+                    response_headers = excluded.response_headers,
+                    downloaded_files = excluded.downloaded_files
+            ''', (
+                result.url,
+                content_hashes['html'],
+                content_hashes['cleaned_html'],
+                content_hashes['markdown'],
+                content_hashes['extracted_content'],
+                result.success,
+                json.dumps(result.media),
+                json.dumps(result.links),
+                json.dumps(result.metadata or {}),
+                content_hashes['screenshot'],
+                json.dumps(result.response_headers or {}),
+                json.dumps(result.downloaded_files or [])
+            ))
+
+        try:
+            await self.execute_with_retry(_cache)
+        except Exception as e:
+            self.logger.error(
+                message="Error caching URL: {error}",
+                tag="ERROR",
+                force_verbose=True,
+                params={"error": str(e)}
+            )
+            
+
+    async def aget_total_count(self) -> int:
+        """Get total number of cached URLs"""
+        async def _count(db):
+            async with db.execute('SELECT COUNT(*) FROM crawled_data') as cursor:
+                result = await cursor.fetchone()
+                return result[0] if result else 0
+
+        try:
+            return await self.execute_with_retry(_count)
+        except Exception as e:
+            self.logger.error(
+                message="Error getting total count: {error}",
+                tag="ERROR",
+                force_verbose=True,
+                params={"error": str(e)}
+            )
+            return 0
+
+    async def aclear_db(self):
+        """Clear all data from the database"""
+        async def _clear(db):
+            await db.execute('DELETE FROM crawled_data')
+
+        try:
+            await self.execute_with_retry(_clear)
+        except Exception as e:
+            self.logger.error(
+                message="Error clearing database: {error}",
+                tag="ERROR",
+                force_verbose=True,
+                params={"error": str(e)}
+            )
+
+    async def aflush_db(self):
+        """Drop the entire table"""
+        async def _flush(db):
+            await db.execute('DROP TABLE IF EXISTS crawled_data')
+
+        try:
+            await self.execute_with_retry(_flush)
+        except Exception as e:
+            self.logger.error(
+                message="Error flushing database: {error}",
+                tag="ERROR",
+                force_verbose=True,
+                params={"error": str(e)}
+            )
+            
+                
+    async def _store_content(self, content: str, content_type: str) -> str:
+        """Store content in filesystem and return hash"""
+        if not content:
+            return ""
+            
+        content_hash = generate_content_hash(content)
+        file_path = os.path.join(self.content_paths[content_type], content_hash)
+        
+        # Only write if file doesn't exist
+        if not os.path.exists(file_path):
+            async with aiofiles.open(file_path, 'w', encoding='utf-8') as f:
+                await f.write(content)
+                
+        return content_hash
+
+    async def _load_content(self, content_hash: str, content_type: str) -> Optional[str]:
+        """Load content from filesystem by hash"""
+        if not content_hash:
+            return None
+            
+        file_path = os.path.join(self.content_paths[content_type], content_hash)
+        try:
+            async with aiofiles.open(file_path, 'r', encoding='utf-8') as f:
+                return await f.read()
+        except:
+            self.logger.error(
+                message="Failed to load content: {file_path}",
+                tag="ERROR",
+                force_verbose=True,
+                params={"file_path": file_path}
+            )
+            return None
+
+# Create a singleton instance
+async_db_manager = AsyncDatabaseManager()

crawl4ai/async_logger.py

@@ -0,0 +1,231 @@
+from enum import Enum
+from typing import Optional, Dict, Any, Union
+from colorama import Fore, Back, Style, init
+import time
+import os
+from datetime import datetime
+
+class LogLevel(Enum):
+    DEBUG = 1
+    INFO = 2
+    SUCCESS = 3
+    WARNING = 4
+    ERROR = 5
+
+class AsyncLogger:
+    """
+    Asynchronous logger with support for colored console output and file logging.
+    Supports templated messages with colored components.
+    """
+    
+    DEFAULT_ICONS = {
+        'INIT': '→',
+        'READY': '✓',
+        'FETCH': '↓',
+        'SCRAPE': '◆',
+        'EXTRACT': '■',
+        'COMPLETE': '●',
+        'ERROR': '×',
+        'DEBUG': '⋯',
+        'INFO': 'ℹ',
+        'WARNING': '⚠',
+    }
+
+    DEFAULT_COLORS = {
+        LogLevel.DEBUG: Fore.LIGHTBLACK_EX,
+        LogLevel.INFO: Fore.CYAN,
+        LogLevel.SUCCESS: Fore.GREEN,
+        LogLevel.WARNING: Fore.YELLOW,
+        LogLevel.ERROR: Fore.RED,
+    }
+
+    def __init__(
+        self,
+        log_file: Optional[str] = None,
+        log_level: LogLevel = LogLevel.DEBUG,
+        tag_width: int = 10,
+        icons: Optional[Dict[str, str]] = None,
+        colors: Optional[Dict[LogLevel, str]] = None,
+        verbose: bool = True
+    ):
+        """
+        Initialize the logger.
+        
+        Args:
+            log_file: Optional file path for logging
+            log_level: Minimum log level to display
+            tag_width: Width for tag formatting
+            icons: Custom icons for different tags
+            colors: Custom colors for different log levels
+            verbose: Whether to output to console
+        """
+        init()  # Initialize colorama
+        self.log_file = log_file
+        self.log_level = log_level
+        self.tag_width = tag_width
+        self.icons = icons or self.DEFAULT_ICONS
+        self.colors = colors or self.DEFAULT_COLORS
+        self.verbose = verbose
+        
+        # Create log file directory if needed
+        if log_file:
+            os.makedirs(os.path.dirname(os.path.abspath(log_file)), exist_ok=True)
+
+    def _format_tag(self, tag: str) -> str:
+        """Format a tag with consistent width."""
+        return f"[{tag}]".ljust(self.tag_width, ".")
+
+    def _get_icon(self, tag: str) -> str:
+        """Get the icon for a tag, defaulting to info icon if not found."""
+        return self.icons.get(tag, self.icons['INFO'])
+
+    def _write_to_file(self, message: str):
+        """Write a message to the log file if configured."""
+        if self.log_file:
+            timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]
+            with open(self.log_file, 'a', encoding='utf-8') as f:
+                # Strip ANSI color codes for file output
+                clean_message = message.replace(Fore.RESET, '').replace(Style.RESET_ALL, '')
+                for color in vars(Fore).values():
+                    if isinstance(color, str):
+                        clean_message = clean_message.replace(color, '')
+                f.write(f"[{timestamp}] {clean_message}\n")
+
+    def _log(
+        self,
+        level: LogLevel,
+        message: str,
+        tag: str,
+        params: Optional[Dict[str, Any]] = None,
+        colors: Optional[Dict[str, str]] = None,
+        base_color: Optional[str] = None,
+        **kwargs
+    ):
+        """
+        Core logging method that handles message formatting and output.
+        
+        Args:
+            level: Log level for this message
+            message: Message template string
+            tag: Tag for the message
+            params: Parameters to format into the message
+            colors: Color overrides for specific parameters
+            base_color: Base color for the entire message
+        """
+        if level.value < self.log_level.value:
+            return
+
+        # Format the message with parameters if provided
+        if params:
+            try:
+                # First format the message with raw parameters
+                formatted_message = message.format(**params)
+                
+                # Then apply colors if specified
+                if colors:
+                    for key, color in colors.items():
+                        # Find the formatted value in the message and wrap it with color
+                        if key in params:
+                            value_str = str(params[key])
+                            formatted_message = formatted_message.replace(
+                                value_str, 
+                                f"{color}{value_str}{Style.RESET_ALL}"
+                            )
+                            
+            except KeyError as e:
+                formatted_message = f"LOGGING ERROR: Missing parameter {e} in message template"
+                level = LogLevel.ERROR
+        else:
+            formatted_message = message
+
+        # Construct the full log line
+        color = base_color or self.colors[level]
+        log_line = f"{color}{self._format_tag(tag)} {self._get_icon(tag)} {formatted_message}{Style.RESET_ALL}"
+
+        # Output to console if verbose
+        if self.verbose or kwargs.get("force_verbose", False):
+            print(log_line)
+
+        # Write to file if configured
+        self._write_to_file(log_line)
+
+    def debug(self, message: str, tag: str = "DEBUG", **kwargs):
+        """Log a debug message."""
+        self._log(LogLevel.DEBUG, message, tag, **kwargs)
+
+    def info(self, message: str, tag: str = "INFO", **kwargs):
+        """Log an info message."""
+        self._log(LogLevel.INFO, message, tag, **kwargs)
+
+    def success(self, message: str, tag: str = "SUCCESS", **kwargs):
+        """Log a success message."""
+        self._log(LogLevel.SUCCESS, message, tag, **kwargs)
+
+    def warning(self, message: str, tag: str = "WARNING", **kwargs):
+        """Log a warning message."""
+        self._log(LogLevel.WARNING, message, tag, **kwargs)
+
+    def error(self, message: str, tag: str = "ERROR", **kwargs):
+        """Log an error message."""
+        self._log(LogLevel.ERROR, message, tag, **kwargs)
+
+    def url_status(
+        self,
+        url: str,
+        success: bool,
+        timing: float,
+        tag: str = "FETCH",
+        url_length: int = 50
+    ):
+        """
+        Convenience method for logging URL fetch status.
+        
+        Args:
+            url: The URL being processed
+            success: Whether the operation was successful
+            timing: Time taken for the operation
+            tag: Tag for the message
+            url_length: Maximum length for URL in log
+        """
+        self._log(
+            level=LogLevel.SUCCESS if success else LogLevel.ERROR,
+            message="{url:.{url_length}}... | Status: {status} | Time: {timing:.2f}s",
+            tag=tag,
+            params={
+                "url": url,
+                "url_length": url_length,
+                "status": success,
+                "timing": timing
+            },
+            colors={
+                "status": Fore.GREEN if success else Fore.RED,
+                "timing": Fore.YELLOW
+            }
+        )
+
+    def error_status(
+        self,
+        url: str,
+        error: str,
+        tag: str = "ERROR",
+        url_length: int = 50
+    ):
+        """
+        Convenience method for logging error status.
+        
+        Args:
+            url: The URL being processed
+            error: Error message
+            tag: Tag for the message
+            url_length: Maximum length for URL in log
+        """
+        self._log(
+            level=LogLevel.ERROR,
+            message="{url:.{url_length}}... | Error: {error}",
+            tag=tag,
+            params={
+                "url": url,
+                "url_length": url_length,
+                "error": error
+            }
+        )

crawl4ai/async_webcrawler.py

@@ -0,0 +1,833 @@
+import os, sys
+import time
+import warnings
+from enum import Enum
+from colorama import init, Fore, Back, Style
+from pathlib import Path
+from typing import Optional, List, Union
+import json
+import asyncio
+# from contextlib import nullcontext, asynccontextmanager
+from contextlib import asynccontextmanager
+from .models import CrawlResult, MarkdownGenerationResult
+from .async_database import async_db_manager
+from .chunking_strategy import *
+from .content_filter_strategy import *
+from .extraction_strategy import *
+from .async_crawler_strategy import AsyncCrawlerStrategy, AsyncPlaywrightCrawlerStrategy, AsyncCrawlResponse
+from .cache_context import CacheMode, CacheContext, _legacy_to_cache_mode
+from .markdown_generation_strategy import DefaultMarkdownGenerator, MarkdownGenerationStrategy
+from .content_scraping_strategy import WebScrapingStrategy
+from .async_logger import AsyncLogger
+from .async_configs import BrowserConfig, CrawlerRunConfig
+from .config import (
+    MIN_WORD_THRESHOLD, 
+    IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD,
+    URL_LOG_SHORTEN_LENGTH
+)
+from .utils import (
+    sanitize_input_encode,
+    InvalidCSSSelectorError,
+    format_html,
+    fast_format_html,
+    create_box_message
+)
+
+from urllib.parse import urlparse
+import random
+from .__version__ import __version__ as crawl4ai_version
+
+
+class AsyncWebCrawler:
+    """
+    Asynchronous web crawler with flexible caching capabilities.
+    
+    There are two ways to use the crawler:
+
+    1. Using context manager (recommended for simple cases):
+        ```python
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun(url="https://example.com")
+        ```
+
+    2. Using explicit lifecycle management (recommended for long-running applications):
+        ```python
+        crawler = AsyncWebCrawler()
+        await crawler.start()
+        
+        # Use the crawler multiple times
+        result1 = await crawler.arun(url="https://example.com")
+        result2 = await crawler.arun(url="https://another.com")
+        
+        await crawler.close()
+        ```
+    
+    Migration Guide:
+    Old way (deprecated):
+        crawler = AsyncWebCrawler(always_by_pass_cache=True, browser_type="chromium", headless=True)
+    
+    New way (recommended):
+        browser_config = BrowserConfig(browser_type="chromium", headless=True)
+        crawler = AsyncWebCrawler(config=browser_config)
+    
+    
+    Attributes:
+        browser_config (BrowserConfig): Configuration object for browser settings.
+        crawler_strategy (AsyncCrawlerStrategy): Strategy for crawling web pages.
+        logger (AsyncLogger): Logger instance for recording events and errors.
+        always_bypass_cache (bool): Whether to always bypass cache.
+        crawl4ai_folder (str): Directory for storing cache.
+        base_directory (str): Base directory for storing cache.
+        ready (bool): Whether the crawler is ready for use.
+        
+        Methods:
+            start(): Start the crawler explicitly without using context manager.
+            close(): Close the crawler explicitly without using context manager.
+            arun(): Run the crawler for a single source: URL (web, local file, or raw HTML).
+            awarmup(): Perform warmup sequence.
+            arun_many(): Run the crawler for multiple sources.
+            aprocess_html(): Process HTML content.
+    
+    Typical Usage:
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun(url="https://example.com")
+            print(result.markdown)
+            
+        Using configuration:
+        browser_config = BrowserConfig(browser_type="chromium", headless=True)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            crawler_config = CrawlerRunConfig(
+                cache_mode=CacheMode.BYPASS                
+            )
+            result = await crawler.arun(url="https://example.com", config=crawler_config)
+            print(result.markdown)
+    """
+    _domain_last_hit = {}
+
+    def __init__(
+        self,
+        crawler_strategy: Optional[AsyncCrawlerStrategy] = None,
+        config: Optional[BrowserConfig] = None,
+        always_bypass_cache: bool = False,
+        always_by_pass_cache: Optional[bool] = None,  # Deprecated parameter
+        base_directory: str = str(os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home())),
+        thread_safe: bool = False,
+        **kwargs,
+    ):
+        """
+        Initialize the AsyncWebCrawler.
+
+        Args:
+            crawler_strategy: Strategy for crawling web pages. If None, will create AsyncPlaywrightCrawlerStrategy
+            config: Configuration object for browser settings. If None, will be created from kwargs
+            always_bypass_cache: Whether to always bypass cache (new parameter)
+            always_by_pass_cache: Deprecated, use always_bypass_cache instead
+            base_directory: Base directory for storing cache
+            thread_safe: Whether to use thread-safe operations
+            **kwargs: Additional arguments for backwards compatibility
+        """  
+        # Handle browser configuration
+        browser_config = config
+        if browser_config is not None:
+            if any(k in kwargs for k in ["browser_type", "headless", "viewport_width", "viewport_height"]):
+                self.logger.warning(
+                    message="Both browser_config and legacy browser parameters provided. browser_config will take precedence.",
+                    tag="WARNING"
+                )
+        else:
+            # Create browser config from kwargs for backwards compatibility
+            browser_config = BrowserConfig.from_kwargs(kwargs)
+
+        self.browser_config = browser_config
+        
+        # Initialize logger first since other components may need it
+        self.logger = AsyncLogger(
+            log_file=os.path.join(base_directory, ".crawl4ai", "crawler.log"),
+            verbose=self.browser_config.verbose,    
+            tag_width=10
+        )
+
+        
+        # Initialize crawler strategy
+        params = {
+            k:v for k, v in kwargs.items() if k in ['browser_congig', 'logger']
+        }
+        self.crawler_strategy = crawler_strategy or AsyncPlaywrightCrawlerStrategy(
+            browser_config=browser_config,
+            logger=self.logger,
+            **params  # Pass remaining kwargs for backwards compatibility
+        )
+        
+        # If craweler strategy doesnt have logger, use crawler logger
+        if not self.crawler_strategy.logger:
+            self.crawler_strategy.logger = self.logger
+        
+        # Handle deprecated cache parameter
+        if always_by_pass_cache is not None:
+            if kwargs.get("warning", True):
+                warnings.warn(
+                    "'always_by_pass_cache' is deprecated and will be removed in version 0.5.0. "
+                    "Use 'always_bypass_cache' instead. "
+                    "Pass warning=False to suppress this warning.",
+                    DeprecationWarning,
+                    stacklevel=2
+                )
+            self.always_bypass_cache = always_by_pass_cache
+        else:
+            self.always_bypass_cache = always_bypass_cache
+
+        # Thread safety setup
+        self._lock = asyncio.Lock() if thread_safe else None
+        
+        # Initialize directories
+        self.crawl4ai_folder = os.path.join(base_directory, ".crawl4ai")
+        os.makedirs(self.crawl4ai_folder, exist_ok=True)
+        os.makedirs(f"{self.crawl4ai_folder}/cache", exist_ok=True)
+        
+        self.ready = False
+
+    async def start(self):
+        """
+        Start the crawler explicitly without using context manager.
+        This is equivalent to using 'async with' but gives more control over the lifecycle.
+        
+        This method will:
+        1. Initialize the browser and context
+        2. Perform warmup sequence
+        3. Return the crawler instance for method chaining
+        
+        Returns:
+            AsyncWebCrawler: The initialized crawler instance
+        """
+        await self.crawler_strategy.__aenter__()
+        await self.awarmup()
+        return self
+
+    async def close(self):
+        """
+        Close the crawler explicitly without using context manager.
+        This should be called when you're done with the crawler if you used start().
+        
+        This method will:
+        1. Clean up browser resources
+        2. Close any open pages and contexts
+        """
+        await self.crawler_strategy.__aexit__(None, None, None)
+
+    async def __aenter__(self):
+        return await self.start()
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self.close()
+    
+    async def awarmup(self):
+        """
+        Initialize the crawler with warm-up sequence.
+        
+        This method:
+        1. Logs initialization info
+        2. Sets up browser configuration
+        3. Marks the crawler as ready
+        """
+        self.logger.info(f"Crawl4AI {crawl4ai_version}", tag="INIT")
+        self.ready = True
+
+    @asynccontextmanager
+    async def nullcontext(self):
+        """异步空上下文管理器"""
+        yield
+
+    async def arun(
+            self,
+            url: str,
+            config: Optional[CrawlerRunConfig] = None,
+            # Legacy parameters maintained for backwards compatibility
+            word_count_threshold=MIN_WORD_THRESHOLD,
+            extraction_strategy: ExtractionStrategy = None,
+            chunking_strategy: ChunkingStrategy = RegexChunking(),
+            content_filter: RelevantContentFilter = None,
+            cache_mode: Optional[CacheMode] = None,
+            # Deprecated cache parameters
+            bypass_cache: bool = False,
+            disable_cache: bool = False,
+            no_cache_read: bool = False,
+            no_cache_write: bool = False,
+            # Other legacy parameters
+            css_selector: str = None,
+            screenshot: bool = False,
+            pdf: bool = False,
+            user_agent: str = None,
+            verbose=True,
+            **kwargs,
+        ) -> CrawlResult:
+            """
+            Runs the crawler for a single source: URL (web, local file, or raw HTML).
+
+            Migration Guide:
+            Old way (deprecated):
+                result = await crawler.arun(
+                    url="https://example.com",
+                    word_count_threshold=200,
+                    screenshot=True,
+                    ...
+                )
+
+            New way (recommended):
+                config = CrawlerRunConfig(
+                    word_count_threshold=200,
+                    screenshot=True,
+                    ...
+                )
+                result = await crawler.arun(url="https://example.com", crawler_config=config)
+
+            Args:
+                url: The URL to crawl (http://, https://, file://, or raw:)
+                crawler_config: Configuration object controlling crawl behavior
+                [other parameters maintained for backwards compatibility]
+
+            Returns:
+                CrawlResult: The result of crawling and processing
+            """
+            crawler_config = config
+            if not isinstance(url, str) or not url:
+                raise ValueError("Invalid URL, make sure the URL is a non-empty string")
+            
+            async with self._lock or self.nullcontext():
+                try:
+                    # Handle configuration
+                    if crawler_config is not None:
+                        # if any(param is not None for param in [
+                        #     word_count_threshold, extraction_strategy, chunking_strategy,
+                        #     content_filter, cache_mode, css_selector, screenshot, pdf
+                        # ]):
+                        #     self.logger.warning(
+                        #         message="Both crawler_config and legacy parameters provided. crawler_config will take precedence.",
+                        #         tag="WARNING"
+                        #     )
+                        config = crawler_config
+                    else:
+                        # Merge all parameters into a single kwargs dict for config creation
+                        config_kwargs = {
+                            "word_count_threshold": word_count_threshold,
+                            "extraction_strategy": extraction_strategy,
+                            "chunking_strategy": chunking_strategy,
+                            "content_filter": content_filter,
+                            "cache_mode": cache_mode,
+                            "bypass_cache": bypass_cache,
+                            "disable_cache": disable_cache,
+                            "no_cache_read": no_cache_read,
+                            "no_cache_write": no_cache_write,
+                            "css_selector": css_selector,
+                            "screenshot": screenshot,
+                            "pdf": pdf,
+                            "verbose": verbose,
+                            **kwargs
+                        }
+                        config = CrawlerRunConfig.from_kwargs(config_kwargs)
+
+                    # Handle deprecated cache parameters
+                    if any([bypass_cache, disable_cache, no_cache_read, no_cache_write]):
+                        if kwargs.get("warning", True):
+                            warnings.warn(
+                                "Cache control boolean flags are deprecated and will be removed in version 0.5.0. "
+                                "Use 'cache_mode' parameter instead.",
+                                DeprecationWarning,
+                                stacklevel=2
+                            )
+                        
+                        # Convert legacy parameters if cache_mode not provided
+                        if config.cache_mode is None:
+                            config.cache_mode = _legacy_to_cache_mode(
+                                disable_cache=disable_cache,
+                                bypass_cache=bypass_cache,
+                                no_cache_read=no_cache_read,
+                                no_cache_write=no_cache_write
+                            )
+                    
+                    # Default to ENABLED if no cache mode specified
+                    if config.cache_mode is None:
+                        config.cache_mode = CacheMode.ENABLED
+
+                    # Create cache context
+                    cache_context = CacheContext(url, config.cache_mode, self.always_bypass_cache)
+
+                    # Initialize processing variables
+                    async_response: AsyncCrawlResponse = None
+                    cached_result: CrawlResult = None
+                    screenshot_data = None
+                    pdf_data = None
+                    extracted_content = None
+                    start_time = time.perf_counter()
+
+                    # Try to get cached result if appropriate
+                    if cache_context.should_read():
+                        cached_result = await async_db_manager.aget_cached_url(url)
+
+                    if cached_result:
+                        html = sanitize_input_encode(cached_result.html)
+                        extracted_content = sanitize_input_encode(cached_result.extracted_content or "")
+                        extracted_content = None if not extracted_content or extracted_content == "[]" else extracted_content
+                        # If screenshot is requested but its not in cache, then set cache_result to None
+                        screenshot_data = cached_result.screenshot
+                        pdf_data = cached_result.pdf
+                        if config.screenshot and not screenshot or config.pdf and not pdf:
+                            cached_result = None
+
+                        self.logger.url_status(
+                            url=cache_context.display_url,
+                            success=bool(html),
+                            timing=time.perf_counter() - start_time,
+                            tag="FETCH"
+                        )
+
+                    # Fetch fresh content if needed
+                    if not cached_result or not html:
+                        t1 = time.perf_counter()
+                        
+                        if user_agent:
+                            self.crawler_strategy.update_user_agent(user_agent)
+                        
+                        # Pass config to crawl method
+                        async_response = await self.crawler_strategy.crawl(
+                            url,
+                            config=config  # Pass the entire config object
+                        )
+                        
+                        html = sanitize_input_encode(async_response.html)
+                        screenshot_data = async_response.screenshot
+                        pdf_data = async_response.pdf_data
+                        
+                        t2 = time.perf_counter()
+                        self.logger.url_status(
+                            url=cache_context.display_url,
+                            success=bool(html),
+                            timing=t2 - t1,
+                            tag="FETCH"
+                        )
+
+                        # Process the HTML content
+                        crawl_result = await self.aprocess_html(
+                            url=url,
+                            html=html,
+                            extracted_content=extracted_content,
+                            config=config,  # Pass the config object instead of individual parameters
+                            screenshot=screenshot_data,
+                            pdf_data=pdf_data,
+                            verbose=config.verbose,
+                            is_raw_html = True if url.startswith("raw:") else False,
+                            **kwargs
+                        )
+
+                        crawl_result.status_code = async_response.status_code
+                        crawl_result.response_headers = async_response.response_headers
+                        crawl_result.downloaded_files = async_response.downloaded_files
+                        crawl_result.ssl_certificate = async_response.ssl_certificate  # Add SSL certificate
+
+                        # # Check and set values from async_response to crawl_result
+                        # try:
+                        #     for key in vars(async_response):
+                        #         if hasattr(crawl_result, key):
+                        #             value = getattr(async_response, key, None)
+                        #             current_value = getattr(crawl_result, key, None)
+                        #             if value is not None and not current_value:
+                        #                 try:
+                        #                     setattr(crawl_result, key, value)
+                        #                 except Exception as e:
+                        #                     self.logger.warning(
+                        #                         message=f"Failed to set attribute {key}: {str(e)}",
+                        #                         tag="WARNING"
+                        #                     )
+                        # except Exception as e:
+                        #     self.logger.warning(
+                        #         message=f"Error copying response attributes: {str(e)}",
+                        #         tag="WARNING"
+                        #     )
+
+                        crawl_result.success = bool(html)
+                        crawl_result.session_id = getattr(config, 'session_id', None)
+
+                        self.logger.success(
+                            message="{url:.50}... | Status: {status} | Total: {timing}",
+                            tag="COMPLETE",
+                            params={
+                                "url": cache_context.display_url,
+                                "status": crawl_result.success,
+                                "timing": f"{time.perf_counter() - start_time:.2f}s"
+                            },
+                            colors={
+                                "status": Fore.GREEN if crawl_result.success else Fore.RED,
+                                "timing": Fore.YELLOW
+                            }
+                        )
+
+                        # Update cache if appropriate
+                        if cache_context.should_write() and not bool(cached_result):
+                            await async_db_manager.acache_url(crawl_result)
+
+                        return crawl_result
+
+                    else:
+                        self.logger.success(
+                            message="{url:.50}... | Status: {status} | Total: {timing}",
+                            tag="COMPLETE",
+                            params={
+                                "url": cache_context.display_url,
+                                "status": True,
+                                "timing": f"{time.perf_counter() - start_time:.2f}s"
+                            },
+                            colors={
+                                "status": Fore.GREEN,
+                                "timing": Fore.YELLOW
+                            }
+                        )
+
+                        cached_result.success = bool(html)
+                        cached_result.session_id = getattr(config, 'session_id', None)
+                        return cached_result
+
+                except Exception as e:
+                    error_context = get_error_context(sys.exc_info())
+                
+                    error_message = (
+                        f"Unexpected error in _crawl_web at line {error_context['line_no']} "
+                        f"in {error_context['function']} ({error_context['filename']}):\n"
+                        f"Error: {str(e)}\n\n"
+                        f"Code context:\n{error_context['code_context']}"
+                    )
+                    # if not hasattr(e, "msg"):
+                    #     e.msg = str(e)
+                    
+                    self.logger.error_status(
+                        url=url,
+                        error=create_box_message(error_message, type="error"),
+                        tag="ERROR"
+                    )
+                    
+                    return CrawlResult(
+                        url=url,
+                        html="",
+                        success=False,
+                        error_message=error_message
+                    )
+
+    async def aprocess_html(
+            self,
+            url: str,
+            html: str,
+            extracted_content: str,
+            config: CrawlerRunConfig,
+            screenshot: str,
+            pdf_data: str,
+            verbose: bool,
+            **kwargs,
+        ) -> CrawlResult:
+            """
+            Process HTML content using the provided configuration.
+            
+            Args:
+                url: The URL being processed
+                html: Raw HTML content
+                extracted_content: Previously extracted content (if any)
+                config: Configuration object controlling processing behavior
+                screenshot: Screenshot data (if any)
+                pdf_data: PDF data (if any)
+                verbose: Whether to enable verbose logging
+                **kwargs: Additional parameters for backwards compatibility
+            
+            Returns:
+                CrawlResult: Processed result containing extracted and formatted content
+            """
+            try:
+                _url = url if not kwargs.get("is_raw_html", False) else "Raw HTML"
+                t1 = time.perf_counter()
+
+                # Initialize scraping strategy
+                scrapping_strategy = WebScrapingStrategy(logger=self.logger)
+
+                # Process HTML content
+                params = {k:v for k, v in config.to_dict().items() if k not in ["url"]}
+                # add keys from kwargs to params that doesn't exist in params
+                params.update({k:v for k, v in kwargs.items() if k not in params.keys()})
+                
+                result = scrapping_strategy.scrap(
+                    url,
+                    html,
+                    **params,
+                    # word_count_threshold=config.word_count_threshold,
+                    # css_selector=config.css_selector,
+                    # only_text=config.only_text,
+                    # image_description_min_word_threshold=config.image_description_min_word_threshold,
+                    # content_filter=config.content_filter,
+                    # **kwargs
+                )
+
+                if result is None:
+                    raise ValueError(f"Process HTML, Failed to extract content from the website: {url}")
+
+            except InvalidCSSSelectorError as e:
+                raise ValueError(str(e))
+            except Exception as e:
+                raise ValueError(f"Process HTML, Failed to extract content from the website: {url}, error: {str(e)}")
+
+       
+
+            # Extract results
+            cleaned_html = sanitize_input_encode(result.get("cleaned_html", ""))
+            fit_markdown = sanitize_input_encode(result.get("fit_markdown", ""))
+            fit_html = sanitize_input_encode(result.get("fit_html", ""))
+            media = result.get("media", [])
+            links = result.get("links", [])
+            metadata = result.get("metadata", {})
+
+            # Markdown Generation
+            markdown_generator: Optional[MarkdownGenerationStrategy] = config.markdown_generator or DefaultMarkdownGenerator()
+            
+            # Uncomment if by default we want to use PruningContentFilter
+            # if not config.content_filter and not markdown_generator.content_filter:
+            #     markdown_generator.content_filter = PruningContentFilter()
+            
+            markdown_result: MarkdownGenerationResult = markdown_generator.generate_markdown(
+                cleaned_html=cleaned_html,
+                base_url=url,
+                # html2text_options=kwargs.get('html2text', {})
+            )
+            markdown_v2 = markdown_result
+            markdown = sanitize_input_encode(markdown_result.raw_markdown)
+
+            # Log processing completion
+            self.logger.info(
+                message="Processed {url:.50}... | Time: {timing}ms",
+                tag="SCRAPE",
+                params={
+                    "url": _url,
+                    "timing": int((time.perf_counter() - t1) * 1000)
+                }
+            )
+
+            # Handle content extraction if needed
+            if (extracted_content is None and 
+                config.extraction_strategy and 
+                config.chunking_strategy and 
+                not isinstance(config.extraction_strategy, NoExtractionStrategy)):
+                
+                t1 = time.perf_counter()
+                
+                # Choose content based on input_format
+                content_format = config.extraction_strategy.input_format
+                if content_format == "fit_markdown" and not markdown_result.fit_markdown:
+                    self.logger.warning(
+                        message="Fit markdown requested but not available. Falling back to raw markdown.",
+                        tag="EXTRACT",
+                        params={"url": _url}
+                    )
+                    content_format = "markdown"
+
+                content = {
+                    "markdown": markdown,
+                    "html": html,
+                    "fit_markdown": markdown_result.raw_markdown
+                }.get(content_format, markdown)
+                
+                # Use IdentityChunking for HTML input, otherwise use provided chunking strategy
+                chunking = IdentityChunking() if content_format == "html" else config.chunking_strategy
+                sections = chunking.chunk(content)
+                extracted_content = config.extraction_strategy.run(url, sections)
+                extracted_content = json.dumps(extracted_content, indent=4, default=str, ensure_ascii=False)
+
+                # Log extraction completion
+                self.logger.info(
+                    message="Completed for {url:.50}... | Time: {timing}s",
+                    tag="EXTRACT",
+                    params={
+                        "url": _url,
+                        "timing": time.perf_counter() - t1
+                    }
+                )
+
+            # Handle screenshot and PDF data
+            screenshot_data = None if not screenshot else screenshot
+            pdf_data = None if not pdf_data else pdf_data
+
+            # Apply HTML formatting if requested
+            if config.prettiify:
+                cleaned_html = fast_format_html(cleaned_html)
+
+            # Return complete crawl result
+            return CrawlResult(
+                url=url,
+                html=html,
+                cleaned_html=cleaned_html,
+                markdown_v2=markdown_v2,
+                markdown=markdown,
+                fit_markdown=fit_markdown,
+                fit_html=fit_html,
+                media=media,
+                links=links,
+                metadata=metadata,
+                screenshot=screenshot_data,
+                pdf=pdf_data,
+                extracted_content=extracted_content,
+                success=True,
+                error_message="",
+            )    
+
+    async def arun_many(
+            self,
+            urls: List[str],
+            config: Optional[CrawlerRunConfig] = None,
+            # Legacy parameters maintained for backwards compatibility
+            word_count_threshold=MIN_WORD_THRESHOLD,
+            extraction_strategy: ExtractionStrategy = None,
+            chunking_strategy: ChunkingStrategy = RegexChunking(),
+            content_filter: RelevantContentFilter = None,
+            cache_mode: Optional[CacheMode] = None,
+            bypass_cache: bool = False,
+            css_selector: str = None,
+            screenshot: bool = False,
+            pdf: bool = False,
+            user_agent: str = None,
+            verbose=True,
+            **kwargs,
+        ) -> List[CrawlResult]:
+            """
+            Runs the crawler for multiple URLs concurrently.
+
+            Migration Guide:
+            Old way (deprecated):
+                results = await crawler.arun_many(
+                    urls,
+                    word_count_threshold=200,
+                    screenshot=True,
+                    ...
+                )
+            
+            New way (recommended):
+                config = CrawlerRunConfig(
+                    word_count_threshold=200,
+                    screenshot=True,
+                    ...
+                )
+                results = await crawler.arun_many(urls, crawler_config=config)
+
+            Args:
+                urls: List of URLs to crawl
+                crawler_config: Configuration object controlling crawl behavior for all URLs
+                [other parameters maintained for backwards compatibility]
+            
+            Returns:
+                List[CrawlResult]: Results for each URL
+            """
+            crawler_config = config
+            # Handle configuration
+            if crawler_config is not None:
+                if any(param is not None for param in [
+                    word_count_threshold, extraction_strategy, chunking_strategy,
+                    content_filter, cache_mode, css_selector, screenshot, pdf
+                ]):
+                    self.logger.warning(
+                        message="Both crawler_config and legacy parameters provided. crawler_config will take precedence.",
+                        tag="WARNING"
+                    )
+                config = crawler_config
+            else:
+                # Merge all parameters into a single kwargs dict for config creation
+                config_kwargs = {
+                    "word_count_threshold": word_count_threshold,
+                    "extraction_strategy": extraction_strategy,
+                    "chunking_strategy": chunking_strategy,
+                    "content_filter": content_filter,
+                    "cache_mode": cache_mode,
+                    "bypass_cache": bypass_cache,
+                    "css_selector": css_selector,
+                    "screenshot": screenshot,
+                    "pdf": pdf,
+                    "verbose": verbose,
+                    **kwargs
+                }
+                config = CrawlerRunConfig.from_kwargs(config_kwargs)
+
+            if bypass_cache:
+                if kwargs.get("warning", True):
+                    warnings.warn(
+                        "'bypass_cache' is deprecated and will be removed in version 0.5.0. "
+                        "Use 'cache_mode=CacheMode.BYPASS' instead. "
+                        "Pass warning=False to suppress this warning.",
+                        DeprecationWarning,
+                        stacklevel=2
+                    )
+                if config.cache_mode is None:
+                    config.cache_mode = CacheMode.BYPASS
+
+            semaphore_count = config.semaphore_count or 5
+            semaphore = asyncio.Semaphore(semaphore_count)
+
+            async def crawl_with_semaphore(url):
+                # Handle rate limiting per domain
+                domain = urlparse(url).netloc
+                current_time = time.time()
+                
+                self.logger.debug(
+                    message="Started task for {url:.50}...",
+                    tag="PARALLEL",
+                    params={"url": url}
+                )
+
+                # Get delay settings from config
+                mean_delay = config.mean_delay
+                max_range = config.max_range
+                
+                # Apply rate limiting
+                if domain in self._domain_last_hit:
+                    time_since_last = current_time - self._domain_last_hit[domain]
+                    if time_since_last < mean_delay:
+                        delay = mean_delay + random.uniform(0, max_range)
+                        await asyncio.sleep(delay)
+                
+                self._domain_last_hit[domain] = current_time
+
+                async with semaphore:
+                    return await self.arun(
+                        url,
+                        crawler_config=config,  # Pass the entire config object
+                        user_agent=user_agent  # Maintain user_agent override capability
+                    )
+
+            # Log start of concurrent crawling
+            self.logger.info(
+                message="Starting concurrent crawling for {count} URLs...",
+                tag="INIT",
+                params={"count": len(urls)}
+            )
+
+            # Execute concurrent crawls
+            start_time = time.perf_counter()
+            tasks = [crawl_with_semaphore(url) for url in urls]
+            results = await asyncio.gather(*tasks, return_exceptions=True)
+            end_time = time.perf_counter()
+
+            # Log completion
+            self.logger.success(
+                message="Concurrent crawling completed for {count} URLs | Total time: {timing}",
+                tag="COMPLETE",
+                params={
+                    "count": len(urls),
+                    "timing": f"{end_time - start_time:.2f}s"
+                },
+                colors={
+                    "timing": Fore.YELLOW
+                }
+            )
+
+            return [result if not isinstance(result, Exception) else str(result) for result in results]
+
+    async def aclear_cache(self):
+        """Clear the cache database."""
+        await async_db_manager.cleanup()
+
+    async def aflush_cache(self):
+        """Flush the cache database."""
+        await async_db_manager.aflush_db()
+
+    async def aget_cache_size(self):
+        """Get the total number of cached items."""
+        return await async_db_manager.aget_total_count()

crawl4ai/cache_context.py

@@ -0,0 +1,115 @@
+from enum import Enum
+
+
+class CacheMode(Enum):
+    """
+    Defines the caching behavior for web crawling operations.
+    
+    Modes:
+    - ENABLED: Normal caching behavior (read and write)
+    - DISABLED: No caching at all
+    - READ_ONLY: Only read from cache, don't write
+    - WRITE_ONLY: Only write to cache, don't read
+    - BYPASS: Bypass cache for this operation
+    """
+    ENABLED = "enabled"
+    DISABLED = "disabled"
+    READ_ONLY = "read_only"
+    WRITE_ONLY = "write_only"
+    BYPASS = "bypass"
+
+
+class CacheContext:
+    """
+    Encapsulates cache-related decisions and URL handling.
+    
+    This class centralizes all cache-related logic and URL type checking,
+    making the caching behavior more predictable and maintainable.
+    
+    Attributes:
+        url (str): The URL being processed.
+        cache_mode (CacheMode): The cache mode for the current operation.
+        always_bypass (bool): If True, bypasses caching for this operation.
+        is_cacheable (bool): True if the URL is cacheable, False otherwise.
+        is_web_url (bool): True if the URL is a web URL, False otherwise.
+        is_local_file (bool): True if the URL is a local file, False otherwise.
+        is_raw_html (bool): True if the URL is raw HTML, False otherwise.
+        _url_display (str): The display name for the URL (web, local file, or raw HTML).
+    """
+    def __init__(self, url: str, cache_mode: CacheMode, always_bypass: bool = False):
+        """
+        Initializes the CacheContext with the provided URL and cache mode.
+        
+        Args:
+            url (str): The URL being processed.
+            cache_mode (CacheMode): The cache mode for the current operation.
+            always_bypass (bool): If True, bypasses caching for this operation.
+        """
+        self.url = url
+        self.cache_mode = cache_mode
+        self.always_bypass = always_bypass
+        self.is_cacheable = url.startswith(('http://', 'https://', 'file://'))
+        self.is_web_url = url.startswith(('http://', 'https://'))
+        self.is_local_file = url.startswith("file://")
+        self.is_raw_html = url.startswith("raw:")
+        self._url_display = url if not self.is_raw_html else "Raw HTML"
+    
+    def should_read(self) -> bool:
+        """
+        Determines if cache should be read based on context.
+        
+        How it works:
+        1. If always_bypass is True or is_cacheable is False, return False.
+        2. If cache_mode is ENABLED or READ_ONLY, return True.
+        
+        Returns:
+            bool: True if cache should be read, False otherwise.
+        """
+        if self.always_bypass or not self.is_cacheable:
+            return False
+        return self.cache_mode in [CacheMode.ENABLED, CacheMode.READ_ONLY]
+    
+    def should_write(self) -> bool:
+        """
+        Determines if cache should be written based on context.
+        
+        How it works:
+        1. If always_bypass is True or is_cacheable is False, return False.
+        2. If cache_mode is ENABLED or WRITE_ONLY, return True.
+        
+        Returns:
+            bool: True if cache should be written, False otherwise.
+        """
+        if self.always_bypass or not self.is_cacheable:
+            return False
+        return self.cache_mode in [CacheMode.ENABLED, CacheMode.WRITE_ONLY]
+    
+    @property
+    def display_url(self) -> str:
+        """Returns the URL in display format."""
+        return self._url_display
+
+
+def _legacy_to_cache_mode(
+    disable_cache: bool = False,
+    bypass_cache: bool = False,
+    no_cache_read: bool = False,
+    no_cache_write: bool = False
+) -> CacheMode:
+    """
+    Converts legacy cache parameters to the new CacheMode enum.
+    
+    This is an internal function to help transition from the old boolean flags
+    to the new CacheMode system.
+    """
+    if disable_cache:
+        return CacheMode.DISABLED
+    if bypass_cache:
+        return CacheMode.BYPASS
+    if no_cache_read and no_cache_write:
+        return CacheMode.DISABLED
+    if no_cache_read:
+        return CacheMode.WRITE_ONLY
+    if no_cache_write:
+        return CacheMode.READ_ONLY
+    return CacheMode.ENABLED

crawl4ai/chunking_strategy.py

@@ -0,0 +1,231 @@
+from abc import ABC, abstractmethod
+import re
+from collections import Counter
+import string
+from .model_loader import load_nltk_punkt
+from .utils import *
+
+# Define the abstract base class for chunking strategies
+class ChunkingStrategy(ABC):
+    """
+    Abstract base class for chunking strategies.
+    """
+    
+    @abstractmethod
+    def chunk(self, text: str) -> list:
+        """
+        Abstract method to chunk the given text.
+        
+        Args:
+            text (str): The text to chunk.
+        
+        Returns:
+            list: A list of chunks.
+        """
+        pass
+
+# Create an identity chunking strategy f(x) = [x]
+class IdentityChunking(ChunkingStrategy):
+    """
+    Chunking strategy that returns the input text as a single chunk.
+    """
+    def chunk(self, text: str) -> list:
+        return [text]
+
+# Regex-based chunking
+class RegexChunking(ChunkingStrategy):
+    """
+    Chunking strategy that splits text based on regular expression patterns.
+    """
+    def __init__(self, patterns=None, **kwargs):
+        """
+        Initialize the RegexChunking object.
+        
+        Args:
+            patterns (list): A list of regular expression patterns to split text.
+        """
+        if patterns is None:
+            patterns = [r'\n\n']  # Default split pattern
+        self.patterns = patterns
+
+    def chunk(self, text: str) -> list:
+        paragraphs = [text]
+        for pattern in self.patterns:
+            new_paragraphs = []
+            for paragraph in paragraphs:
+                new_paragraphs.extend(re.split(pattern, paragraph))
+            paragraphs = new_paragraphs
+        return paragraphs
+    
+# NLP-based sentence chunking 
+class NlpSentenceChunking(ChunkingStrategy):
+    """
+    Chunking strategy that splits text into sentences using NLTK's sentence tokenizer.
+    """ 
+    def __init__(self, **kwargs):
+        """
+        Initialize the NlpSentenceChunking object.
+        """
+        load_nltk_punkt()
+        
+
+    def chunk(self, text: str) -> list:
+        # Improved regex for sentence splitting
+        # sentence_endings = re.compile(
+        #     r'(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<![A-Z][A-Z]\.)(?<![A-Za-z]\.)(?<=\.|\?|\!|\n)\s'
+        # )
+        # sentences = sentence_endings.split(text)
+        # sens =  [sent.strip() for sent in sentences if sent]            
+        from nltk.tokenize import sent_tokenize
+        sentences = sent_tokenize(text)
+        sens =  [sent.strip() for sent in sentences]        
+        
+        return list(set(sens))
+    
+# Topic-based segmentation using TextTiling
+class TopicSegmentationChunking(ChunkingStrategy):
+    """
+    Chunking strategy that segments text into topics using NLTK's TextTilingTokenizer.
+    
+    How it works:
+    1. Segment the text into topics using TextTilingTokenizer
+    2. Extract keywords for each topic segment
+    """
+    
+    def __init__(self, num_keywords=3, **kwargs):
+        """
+        Initialize the TopicSegmentationChunking object.
+        
+        Args:
+            num_keywords (int): The number of keywords to extract for each topic segment.
+        """
+        import nltk as nl
+        self.tokenizer = nl.tokenize.TextTilingTokenizer()
+        self.num_keywords = num_keywords
+
+    def chunk(self, text: str) -> list:
+        # Use the TextTilingTokenizer to segment the text
+        segmented_topics = self.tokenizer.tokenize(text)
+        return segmented_topics
+
+    def extract_keywords(self, text: str) -> list:
+        # Tokenize and remove stopwords and punctuation
+        import nltk as nl
+        tokens = nl.toknize.word_tokenize(text)
+        tokens = [token.lower() for token in tokens if token not in nl.corpus.stopwords.words('english') and token not in string.punctuation]
+
+        # Calculate frequency distribution
+        freq_dist = Counter(tokens)
+        keywords = [word for word, freq in freq_dist.most_common(self.num_keywords)]
+        return keywords
+
+    def chunk_with_topics(self, text: str) -> list:
+        # Segment the text into topics
+        segments = self.chunk(text)
+        # Extract keywords for each topic segment
+        segments_with_topics = [(segment, self.extract_keywords(segment)) for segment in segments]
+        return segments_with_topics
+    
+# Fixed-length word chunks
+class FixedLengthWordChunking(ChunkingStrategy):
+    """
+    Chunking strategy that splits text into fixed-length word chunks.
+    
+    How it works:
+    1. Split the text into words
+    2. Create chunks of fixed length
+    3. Return the list of chunks
+    """
+    def __init__(self, chunk_size=100, **kwargs):
+        """
+        Initialize the fixed-length word chunking strategy with the given chunk size.
+        
+        Args:
+            chunk_size (int): The size of each chunk in words.
+        """
+        self.chunk_size = chunk_size
+
+    def chunk(self, text: str) -> list:
+        words = text.split()
+        return [' '.join(words[i:i + self.chunk_size]) for i in range(0, len(words), self.chunk_size)]
+    
+# Sliding window chunking
+class SlidingWindowChunking(ChunkingStrategy):
+    """
+    Chunking strategy that splits text into overlapping word chunks.
+    
+    How it works:
+    1. Split the text into words
+    2. Create chunks of fixed length
+    3. Return the list of chunks
+    """
+    def __init__(self, window_size=100, step=50, **kwargs):
+        """
+        Initialize the sliding window chunking strategy with the given window size and
+        step size.
+        
+        Args:
+            window_size (int): The size of the sliding window in words.
+            step (int): The step size for sliding the window in words.
+        """
+        self.window_size = window_size
+        self.step = step
+
+    def chunk(self, text: str) -> list:
+        words = text.split()
+        chunks = []
+        
+        if len(words) <= self.window_size:
+            return [text]
+        
+        for i in range(0, len(words) - self.window_size + 1, self.step):
+            chunk = ' '.join(words[i:i + self.window_size])
+            chunks.append(chunk)
+        
+        # Handle the last chunk if it doesn't align perfectly
+        if i + self.window_size < len(words):
+            chunks.append(' '.join(words[-self.window_size:]))
+        
+        return chunks
+    
+class OverlappingWindowChunking(ChunkingStrategy):
+    """
+    Chunking strategy that splits text into overlapping word chunks.
+    
+    How it works:
+    1. Split the text into words using whitespace
+    2. Create chunks of fixed length equal to the window size
+    3. Slide the window by the overlap size
+    4. Return the list of chunks
+    """
+    def __init__(self, window_size=1000, overlap=100, **kwargs):
+        """
+        Initialize the overlapping window chunking strategy with the given window size and
+        overlap size.
+        
+        Args:
+            window_size (int): The size of the window in words.
+            overlap (int): The size of the overlap between consecutive chunks in words.
+        """
+        self.window_size = window_size
+        self.overlap = overlap
+
+    def chunk(self, text: str) -> list:
+        words = text.split()
+        chunks = []
+        
+        if len(words) <= self.window_size:
+            return [text]
+        
+        start = 0
+        while start < len(words):
+            end = start + self.window_size
+            chunk = ' '.join(words[start:end])
+            chunks.append(chunk)
+            
+            if end >= len(words):
+                break
+            
+            start = end - self.overlap
+        
+        return chunks

crawl4ai/cli.py

@@ -0,0 +1,105 @@
+import click
+import sys
+import asyncio
+from typing import List
+from .docs_manager import DocsManager
+from .async_logger import AsyncLogger
+
+logger = AsyncLogger(verbose=True)
+docs_manager = DocsManager(logger)
+
+def print_table(headers: List[str], rows: List[List[str]], padding: int = 2):
+    """Print formatted table with headers and rows"""
+    widths = [max(len(str(cell)) for cell in col) for col in zip(headers, *rows)]
+    border = '+' + '+'.join('-' * (w + 2 * padding) for w in widths) + '+'
+    
+    def format_row(row):
+        return '|' + '|'.join(f"{' ' * padding}{str(cell):<{w}}{' ' * padding}" 
+                             for cell, w in zip(row, widths)) + '|'
+    
+    click.echo(border)
+    click.echo(format_row(headers))
+    click.echo(border)
+    for row in rows:
+        click.echo(format_row(row))
+    click.echo(border)
+
+@click.group()
+def cli():
+    """Crawl4AI Command Line Interface"""
+    pass
+
+@cli.group()
+def docs():
+    """Documentation operations"""
+    pass
+
+@docs.command()
+@click.argument('sections', nargs=-1)
+@click.option('--mode', type=click.Choice(['extended', 'condensed']), default='extended')
+def combine(sections: tuple, mode: str):
+    """Combine documentation sections"""
+    try:
+        asyncio.run(docs_manager.ensure_docs_exist())
+        click.echo(docs_manager.generate(sections, mode))
+    except Exception as e:
+        logger.error(str(e), tag="ERROR")
+        sys.exit(1)
+
+@docs.command()
+@click.argument('query')
+@click.option('--top-k', '-k', default=5)
+@click.option('--build-index', is_flag=True, help='Build index if missing')
+def search(query: str, top_k: int, build_index: bool):
+    """Search documentation"""
+    try:
+        result = docs_manager.search(query, top_k)
+        if result == "No search index available. Call build_search_index() first.":
+            if build_index or click.confirm('No search index found. Build it now?'):
+                asyncio.run(docs_manager.llm_text.generate_index_files())
+                result = docs_manager.search(query, top_k)
+        click.echo(result)
+    except Exception as e:
+        click.echo(f"Error: {str(e)}", err=True)
+        sys.exit(1)
+
+@docs.command()
+def update():
+    """Update docs from GitHub"""
+    try:
+        asyncio.run(docs_manager.fetch_docs())
+        click.echo("Documentation updated successfully")
+    except Exception as e:
+        click.echo(f"Error: {str(e)}", err=True)
+        sys.exit(1)
+
+@docs.command()
+@click.option('--force-facts', is_flag=True, help='Force regenerate fact files')
+@click.option('--clear-cache', is_flag=True, help='Clear BM25 cache')
+def index(force_facts: bool, clear_cache: bool):
+    """Build or rebuild search indexes"""
+    try:
+        asyncio.run(docs_manager.ensure_docs_exist())
+        asyncio.run(docs_manager.llm_text.generate_index_files(
+            force_generate_facts=force_facts,
+            clear_bm25_cache=clear_cache
+        ))
+        click.echo("Search indexes built successfully")
+    except Exception as e:
+        click.echo(f"Error: {str(e)}", err=True)
+        sys.exit(1)
+
+# Add docs list command
+@docs.command()
+def list():
+    """List available documentation sections"""
+    try:
+        sections = docs_manager.list()
+        print_table(["Sections"], [[section] for section in sections])
+        
+    except Exception as e:
+        click.echo(f"Error: {str(e)}", err=True)
+        sys.exit(1)
+
+if __name__ == '__main__':
+    cli()

crawl4ai/config.py

@@ -0,0 +1,64 @@
+import os
+from dotenv import load_dotenv
+
+load_dotenv()  # Load environment variables from .env file
+
+# Default provider, ONLY used when the extraction strategy is LLMExtractionStrategy
+DEFAULT_PROVIDER = "openai/gpt-4o-mini"
+MODEL_REPO_BRANCH = "new-release-0.0.2"
+# Provider-model dictionary, ONLY used when the extraction strategy is LLMExtractionStrategy
+PROVIDER_MODELS = {
+    "ollama/llama3": "no-token-needed", # Any model from Ollama no need for API token
+    "groq/llama3-70b-8192": os.getenv("GROQ_API_KEY"),
+    "groq/llama3-8b-8192": os.getenv("GROQ_API_KEY"),
+    "openai/gpt-4o-mini": os.getenv("OPENAI_API_KEY"),
+    "openai/gpt-4o": os.getenv("OPENAI_API_KEY"),
+    "openai/o1-mini": os.getenv("OPENAI_API_KEY"),
+    "openai/o1-preview": os.getenv("OPENAI_API_KEY"),
+    "anthropic/claude-3-haiku-20240307": os.getenv("ANTHROPIC_API_KEY"),
+    "anthropic/claude-3-opus-20240229": os.getenv("ANTHROPIC_API_KEY"),
+    "anthropic/claude-3-sonnet-20240229": os.getenv("ANTHROPIC_API_KEY"),
+    "anthropic/claude-3-5-sonnet-20240620": os.getenv("ANTHROPIC_API_KEY"),
+}
+
+# Chunk token threshold
+CHUNK_TOKEN_THRESHOLD = 2 ** 11 # 2048 tokens
+OVERLAP_RATE = 0.1
+WORD_TOKEN_RATE = 1.3
+
+# Threshold for the minimum number of word in a HTML tag to be considered 
+MIN_WORD_THRESHOLD = 1
+IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD = 1
+
+IMPORTANT_ATTRS = ['src', 'href', 'alt', 'title', 'width', 'height'] 
+ONLY_TEXT_ELIGIBLE_TAGS = ['b', 'i', 'u', 'span', 'del', 'ins', 'sub', 'sup', 'strong', 'em', 'code', 'kbd', 'var', 's', 'q', 'abbr', 'cite', 'dfn', 'time', 'small', 'mark']
+SOCIAL_MEDIA_DOMAINS = [
+                            'facebook.com',
+                            'twitter.com',
+                            'x.com',
+                            'linkedin.com',
+                            'instagram.com',
+                            'pinterest.com',
+                            'tiktok.com',
+                            'snapchat.com',
+                            'reddit.com',
+                        ]
+
+# Threshold for the Image extraction - Range is 1 to 6
+# Images are scored based on point based system, to filter based on usefulness. Points are assigned
+# to each image based on the following aspects.
+# If either height or width exceeds 150px
+# If image size is greater than 10Kb
+# If alt property is set
+# If image format is in jpg, png or webp
+# If image is in the first half of the total images extracted from the page
+IMAGE_SCORE_THRESHOLD = 2
+
+MAX_METRICS_HISTORY = 1000
+
+NEED_MIGRATION = True
+URL_LOG_SHORTEN_LENGTH = 30
+SHOW_DEPRECATION_WARNINGS = True
+SCREENSHOT_HEIGHT_TRESHOLD = 10000
+PAGE_TIMEOUT=60000
+DOWNLOAD_PAGE_TIMEOUT=60000

crawl4ai/content_filter_strategy.py

@@ -0,0 +1,627 @@
+import re
+from bs4 import BeautifulSoup, Tag
+from typing import List, Tuple, Dict
+from rank_bm25 import BM25Okapi
+from time import perf_counter
+from collections import deque
+from bs4 import BeautifulSoup, NavigableString, Tag, Comment
+from .utils import clean_tokens
+from abc import ABC, abstractmethod
+import math
+from snowballstemmer import stemmer
+class RelevantContentFilter(ABC):
+    """Abstract base class for content filtering strategies"""
+    def __init__(self, user_query: str = None):
+        self.user_query = user_query
+        self.included_tags = {
+            # Primary structure
+            'article', 'main', 'section', 'div', 
+            # List structures
+            'ul', 'ol', 'li', 'dl', 'dt', 'dd',
+            # Text content
+            'p', 'span', 'blockquote', 'pre', 'code',
+            # Headers
+            'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
+            # Tables
+            'table', 'thead', 'tbody', 'tr', 'td', 'th',
+            # Other semantic elements
+            'figure', 'figcaption', 'details', 'summary',
+            # Text formatting
+            'em', 'strong', 'b', 'i', 'mark', 'small',
+            # Rich content
+            'time', 'address', 'cite', 'q'
+        }
+        self.excluded_tags = {
+            'nav', 'footer', 'header', 'aside', 'script',
+            'style', 'form', 'iframe', 'noscript'
+        }
+        self.header_tags = {'h1', 'h2', 'h3', 'h4', 'h5', 'h6'}
+        self.negative_patterns = re.compile(
+            r'nav|footer|header|sidebar|ads|comment|promo|advert|social|share',
+            re.I
+        )
+        self.min_word_count = 2
+        
+    @abstractmethod
+    def filter_content(self, html: str) -> List[str]:
+        """Abstract method to be implemented by specific filtering strategies"""
+        pass
+    
+    def extract_page_query(self, soup: BeautifulSoup, body: Tag) -> str:
+        """Common method to extract page metadata with fallbacks"""
+        if self.user_query:
+            return self.user_query
+
+        query_parts = []
+        
+        # Title
+        try:
+            title = soup.title.string
+            if title:
+                query_parts.append(title)
+        except Exception:
+            pass
+
+        if soup.find('h1'):
+            query_parts.append(soup.find('h1').get_text())
+            
+        # Meta tags
+        temp = ""
+        for meta_name in ['keywords', 'description']:
+            meta = soup.find('meta', attrs={'name': meta_name})
+            if meta and meta.get('content'):
+                query_parts.append(meta['content'])
+                temp += meta['content']
+                
+        # If still empty, grab first significant paragraph
+        if not temp:
+            # Find the first tag P thatits text contains more than 50 characters
+            for p in body.find_all('p'):
+                if len(p.get_text()) > 150:
+                    query_parts.append(p.get_text()[:150])
+                    break        
+                                
+        return ' '.join(filter(None, query_parts))
+
+    def extract_text_chunks(self, body: Tag, min_word_threshold: int = None) -> List[Tuple[str, str]]:
+        """
+        Extracts text chunks from a BeautifulSoup body element while preserving order.
+        Returns list of tuples (text, tag_name) for classification.
+        
+        Args:
+            body: BeautifulSoup Tag object representing the body element
+            
+        Returns:
+            List of (text, tag_name) tuples
+        """
+        # Tags to ignore - inline elements that shouldn't break text flow
+        INLINE_TAGS = {
+            'a', 'abbr', 'acronym', 'b', 'bdo', 'big', 'br', 'button', 'cite', 'code',
+            'dfn', 'em', 'i', 'img', 'input', 'kbd', 'label', 'map', 'object', 'q',
+            'samp', 'script', 'select', 'small', 'span', 'strong', 'sub', 'sup',
+            'textarea', 'time', 'tt', 'var'
+        }
+        
+        # Tags that typically contain meaningful headers
+        HEADER_TAGS = {'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'header'}
+        
+        chunks = []
+        current_text = []
+        chunk_index = 0
+    
+        def should_break_chunk(tag: Tag) -> bool:
+            """Determine if a tag should cause a break in the current text chunk"""
+            return (
+                tag.name not in INLINE_TAGS
+                and not (tag.name == 'p' and len(current_text) == 0)
+            )
+        
+        # Use deque for efficient push/pop operations
+        stack = deque([(body, False)])
+        
+        while stack:
+            element, visited = stack.pop()
+            
+            if visited:
+                # End of block element - flush accumulated text
+                if current_text and should_break_chunk(element):
+                    text = ' '.join(''.join(current_text).split())
+                    if text:
+                        tag_type = 'header' if element.name in HEADER_TAGS else 'content'
+                        chunks.append((chunk_index, text, tag_type, element))
+                        chunk_index += 1
+                    current_text = []
+                continue
+                
+            if isinstance(element, NavigableString):
+                if str(element).strip():
+                    current_text.append(str(element).strip())
+                continue
+                
+            # Pre-allocate children to avoid multiple list operations
+            children = list(element.children)
+            if not children:
+                continue
+                
+            # Mark block for revisit after processing children
+            stack.append((element, True))
+            
+            # Add children in reverse order for correct processing
+            for child in reversed(children):
+                if isinstance(child, (Tag, NavigableString)):
+                    stack.append((child, False))
+        
+        # Handle any remaining text
+        if current_text:
+            text = ' '.join(''.join(current_text).split())
+            if text:
+                chunks.append((chunk_index, text, 'content', body))
+        
+        if min_word_threshold:
+            chunks = [chunk for chunk in chunks if len(chunk[1].split()) >= min_word_threshold]
+        
+        return chunks    
+
+    def _deprecated_extract_text_chunks(self, soup: BeautifulSoup) -> List[Tuple[int, str, Tag]]:
+        """Common method for extracting text chunks"""
+        _text_cache = {}
+        def fast_text(element: Tag) -> str:
+            elem_id = id(element)
+            if elem_id in _text_cache:
+                return _text_cache[elem_id]
+            texts = []
+            for content in element.contents:
+                if isinstance(content, str):
+                    text = content.strip()
+                    if text:
+                        texts.append(text)
+            result = ' '.join(texts)
+            _text_cache[elem_id] = result
+            return result
+        
+        candidates = []
+        index = 0
+        
+        def dfs(element):
+            nonlocal index
+            if isinstance(element, Tag):
+                if element.name in self.included_tags:
+                    if not self.is_excluded(element):
+                        text = fast_text(element)
+                        word_count = len(text.split())
+                        
+                        # Headers pass through with adjusted minimum
+                        if element.name in self.header_tags:
+                            if word_count >= 3:  # Minimal sanity check for headers
+                                candidates.append((index, text, element))
+                                index += 1
+                        # Regular content uses standard minimum
+                        elif word_count >= self.min_word_count:
+                            candidates.append((index, text, element))
+                            index += 1
+                            
+                for child in element.children:
+                    dfs(child)
+
+        dfs(soup.body if soup.body else soup)
+        return candidates
+
+    def is_excluded(self, tag: Tag) -> bool:
+        """Common method for exclusion logic"""
+        if tag.name in self.excluded_tags:
+            return True
+        class_id = ' '.join(filter(None, [
+            ' '.join(tag.get('class', [])),
+            tag.get('id', '')
+        ]))
+        return bool(self.negative_patterns.search(class_id))
+
+    def clean_element(self, tag: Tag) -> str:
+        """Common method for cleaning HTML elements with minimal overhead"""
+        if not tag or not isinstance(tag, Tag):
+            return ""
+            
+        unwanted_tags = {'script', 'style', 'aside', 'form', 'iframe', 'noscript'}
+        unwanted_attrs = {'style', 'onclick', 'onmouseover', 'align', 'bgcolor', 'class', 'id'}
+        
+        # Use string builder pattern for better performance
+        builder = []
+        
+        def render_tag(elem):
+            if not isinstance(elem, Tag):
+                if isinstance(elem, str):
+                    builder.append(elem.strip())
+                return
+                
+            if elem.name in unwanted_tags:
+                return
+                
+            # Start tag
+            builder.append(f'<{elem.name}')
+            
+            # Add cleaned attributes
+            attrs = {k: v for k, v in elem.attrs.items() if k not in unwanted_attrs}
+            for key, value in attrs.items():
+                builder.append(f' {key}="{value}"')
+                
+            builder.append('>')
+            
+            # Process children
+            for child in elem.children:
+                render_tag(child)
+                
+            # Close tag
+            builder.append(f'</{elem.name}>')
+        
+        try:
+            render_tag(tag)
+            return ''.join(builder)
+        except Exception:
+            return str(tag)  # Fallback to original if anything fails
+
+class BM25ContentFilter(RelevantContentFilter):
+    """
+    Content filtering using BM25 algorithm with priority tag handling.
+    
+    How it works:
+    1. Extracts page metadata with fallbacks.
+    2. Extracts text chunks from the body element.
+    3. Tokenizes the corpus and query.
+    4. Applies BM25 algorithm to calculate scores for each chunk.
+    5. Filters out chunks below the threshold.
+    6. Sorts chunks by score in descending order.
+    7. Returns the top N chunks.
+    
+    Attributes:
+        user_query (str): User query for filtering (optional).
+        bm25_threshold (float): BM25 threshold for filtering (default: 1.0).
+        language (str): Language for stemming (default: 'english').
+        
+        Methods:
+            filter_content(self, html: str, min_word_threshold: int = None)
+    """
+    def __init__(self, user_query: str = None, bm25_threshold: float = 1.0, language: str = 'english'):
+        """
+        Initializes the BM25ContentFilter class, if not provided, falls back to page metadata.
+        
+        Note:
+        If no query is given and no page metadata is available, then it tries to pick up the first significant paragraph.
+        
+        Args:
+            user_query (str): User query for filtering (optional).
+            bm25_threshold (float): BM25 threshold for filtering (default: 1.0).
+            language (str): Language for stemming (default: 'english').
+        """
+        super().__init__(user_query=user_query)
+        self.bm25_threshold = bm25_threshold
+        self.priority_tags = {
+            'h1': 5.0,
+            'h2': 4.0,
+            'h3': 3.0,
+            'title': 4.0,
+            'strong': 2.0,
+            'b': 1.5,
+            'em': 1.5,
+            'blockquote': 2.0,
+            'code': 2.0,
+            'pre': 1.5,
+            'th': 1.5,  # Table headers
+        }
+        self.stemmer = stemmer(language)
+
+    def filter_content(self, html: str, min_word_threshold: int = None) -> List[str]:
+        """
+        Implements content filtering using BM25 algorithm with priority tag handling.
+        
+            Note:
+        This method implements the filtering logic for the BM25ContentFilter class.
+        It takes HTML content as input and returns a list of filtered text chunks.
+        
+        Args:
+            html (str): HTML content to be filtered.
+            min_word_threshold (int): Minimum word threshold for filtering (optional).
+        
+        Returns:
+            List[str]: List of filtered text chunks.
+        """
+        if not html or not isinstance(html, str):
+            return []
+
+        soup = BeautifulSoup(html, 'lxml')
+        
+        # Check if body is present
+        if not soup.body:
+            # Wrap in body tag if missing
+            soup = BeautifulSoup(f'<body>{html}</body>', 'lxml')        
+        body = soup.find('body')
+        
+        query = self.extract_page_query(soup, body)
+        
+        if not query:
+            return []
+            # return [self.clean_element(soup)]
+            
+        candidates = self.extract_text_chunks(body, min_word_threshold)
+
+        if not candidates:
+            return []
+
+        # Tokenize corpus
+        # tokenized_corpus = [chunk.lower().split() for _, chunk, _, _ in candidates]
+        # tokenized_query = query.lower().split()
+                
+        # tokenized_corpus = [[ps.stem(word) for word in chunk.lower().split()] 
+        #                 for _, chunk, _, _ in candidates]
+        # tokenized_query = [ps.stem(word) for word in query.lower().split()]        
+        
+        tokenized_corpus = [[self.stemmer.stemWord(word) for word in chunk.lower().split()] 
+                   for _, chunk, _, _ in candidates]
+        tokenized_query = [self.stemmer.stemWord(word) for word in query.lower().split()]
+
+        # tokenized_corpus = [[self.stemmer.stemWord(word) for word in tokenize_text(chunk.lower())] 
+        #            for _, chunk, _, _ in candidates]
+        # tokenized_query = [self.stemmer.stemWord(word) for word in tokenize_text(query.lower())]
+
+        # Clean from stop words and noise
+        tokenized_corpus = [clean_tokens(tokens) for tokens in tokenized_corpus]
+        tokenized_query = clean_tokens(tokenized_query)
+
+        bm25 = BM25Okapi(tokenized_corpus)
+        scores = bm25.get_scores(tokenized_query)
+
+        # Adjust scores with tag weights
+        adjusted_candidates = []
+        for score, (index, chunk, tag_type, tag) in zip(scores, candidates):
+            tag_weight = self.priority_tags.get(tag.name, 1.0)
+            adjusted_score = score * tag_weight
+            adjusted_candidates.append((adjusted_score, index, chunk, tag))
+
+        # Filter candidates by threshold
+        selected_candidates = [
+            (index, chunk, tag) for adjusted_score, index, chunk, tag in adjusted_candidates
+            if adjusted_score >= self.bm25_threshold
+        ]
+
+        if not selected_candidates:
+            return []
+
+        # Sort selected candidates by original document order
+        selected_candidates.sort(key=lambda x: x[0])
+
+        return [self.clean_element(tag) for _, _, tag in selected_candidates]
+
+class PruningContentFilter(RelevantContentFilter):
+    """
+    Content filtering using pruning algorithm with dynamic threshold.
+    
+    How it works:
+    1. Extracts page metadata with fallbacks.
+    2. Extracts text chunks from the body element.
+    3. Applies pruning algorithm to calculate scores for each chunk.
+    4. Filters out chunks below the threshold.
+    5. Sorts chunks by score in descending order.
+    6. Returns the top N chunks.
+
+    Attributes:
+        user_query (str): User query for filtering (optional), if not provided, falls back to page metadata.
+        min_word_threshold (int): Minimum word threshold for filtering (optional).
+        threshold_type (str): Threshold type for dynamic threshold (default: 'fixed').
+        threshold (float): Fixed threshold value (default: 0.48).
+        
+        Methods:
+            filter_content(self, html: str, min_word_threshold: int = None):
+    """
+    def __init__(self, user_query: str = None, min_word_threshold: int = None, 
+                 threshold_type: str = 'fixed', threshold: float = 0.48):
+        """
+        Initializes the PruningContentFilter class, if not provided, falls back to page metadata.
+        
+        Note:
+        If no query is given and no page metadata is available, then it tries to pick up the first significant paragraph.
+        
+        Args:
+            user_query (str): User query for filtering (optional).
+            min_word_threshold (int): Minimum word threshold for filtering (optional).
+            threshold_type (str): Threshold type for dynamic threshold (default: 'fixed').
+            threshold (float): Fixed threshold value (default: 0.48).
+        """
+        super().__init__(None)
+        self.min_word_threshold = min_word_threshold
+        self.threshold_type = threshold_type
+        self.threshold = threshold
+        
+        # Add tag importance for dynamic threshold
+        self.tag_importance = {
+            'article': 1.5,
+            'main': 1.4,
+            'section': 1.3,
+            'p': 1.2,
+            'h1': 1.4,
+            'h2': 1.3,
+            'h3': 1.2,
+            'div': 0.7,
+            'span': 0.6
+        }
+        
+        # Metric configuration
+        self.metric_config = {
+            'text_density': True,
+            'link_density': True,
+            'tag_weight': True,
+            'class_id_weight': True,
+            'text_length': True,
+        }
+        
+        self.metric_weights = {
+            'text_density': 0.4,
+            'link_density': 0.2,
+            'tag_weight': 0.2,
+            'class_id_weight': 0.1,
+            'text_length': 0.1,
+        }
+        
+        self.tag_weights = {
+            'div': 0.5,
+            'p': 1.0,
+            'article': 1.5,
+            'section': 1.0,
+            'span': 0.3,
+            'li': 0.5,
+            'ul': 0.5,
+            'ol': 0.5,
+            'h1': 1.2,
+            'h2': 1.1,
+            'h3': 1.0,
+            'h4': 0.9,
+            'h5': 0.8,
+            'h6': 0.7,
+        }
+
+    def filter_content(self, html: str, min_word_threshold: int = None) -> List[str]:
+        """
+        Implements content filtering using pruning algorithm with dynamic threshold.
+        
+        Note:
+        This method implements the filtering logic for the PruningContentFilter class.
+        It takes HTML content as input and returns a list of filtered text chunks.
+        
+        Args:
+            html (str): HTML content to be filtered.
+            min_word_threshold (int): Minimum word threshold for filtering (optional).
+        
+        Returns:
+            List[str]: List of filtered text chunks.
+        """
+        if not html or not isinstance(html, str):
+            return []
+            
+        soup = BeautifulSoup(html, 'lxml')
+        if not soup.body:
+            soup = BeautifulSoup(f'<body>{html}</body>', 'lxml')
+        
+        # Remove comments and unwanted tags
+        self._remove_comments(soup)
+        self._remove_unwanted_tags(soup)
+        
+        # Prune tree starting from body
+        body = soup.find('body')
+        self._prune_tree(body)
+        
+        # Extract remaining content as list of HTML strings
+        content_blocks = []
+        for element in body.children:
+            if isinstance(element, str) or not hasattr(element, 'name'):
+                continue
+            if len(element.get_text(strip=True)) > 0:
+                content_blocks.append(str(element))
+                
+        return content_blocks
+
+    def _remove_comments(self, soup):
+        """Removes HTML comments"""
+        for element in soup(text=lambda text: isinstance(text, Comment)):
+            element.extract()
+
+    def _remove_unwanted_tags(self, soup):
+        """Removes unwanted tags"""
+        for tag in self.excluded_tags:
+            for element in soup.find_all(tag):
+                element.decompose()
+
+    def _prune_tree(self, node):
+        """
+        Prunes the tree starting from the given node.
+        
+        Args:
+            node (Tag): The node from which the pruning starts.
+        """
+        if not node or not hasattr(node, 'name') or node.name is None:
+            return
+
+        text_len = len(node.get_text(strip=True))
+        tag_len = len(node.encode_contents().decode('utf-8'))
+        link_text_len = sum(len(s.strip()) for s in (a.string for a in node.find_all('a', recursive=False)) if s)
+
+        metrics = {
+            'node': node,
+            'tag_name': node.name,
+            'text_len': text_len,
+            'tag_len': tag_len,
+            'link_text_len': link_text_len
+        }
+
+        score = self._compute_composite_score(metrics, text_len, tag_len, link_text_len)
+
+        if self.threshold_type == 'fixed':
+            should_remove = score < self.threshold
+        else:  # dynamic
+            tag_importance = self.tag_importance.get(node.name, 0.7)
+            text_ratio = text_len / tag_len if tag_len > 0 else 0
+            link_ratio = link_text_len / text_len if text_len > 0 else 1
+            
+            threshold = self.threshold  # base threshold
+            if tag_importance > 1:
+                threshold *= 0.8
+            if text_ratio > 0.4:
+                threshold *= 0.9
+            if link_ratio > 0.6:
+                threshold *= 1.2
+                
+            should_remove = score < threshold
+
+        if should_remove:
+            node.decompose()
+        else:
+            children = [child for child in node.children if hasattr(child, 'name')]
+            for child in children:
+                self._prune_tree(child)
+
+    def _compute_composite_score(self, metrics, text_len, tag_len, link_text_len):
+        """Computes the composite score"""
+        if self.min_word_threshold:
+            # Get raw text from metrics node - avoid extra processing
+            text = metrics['node'].get_text(strip=True)
+            word_count = text.count(' ') + 1
+            if word_count < self.min_word_threshold:
+                return -1.0  # Guaranteed removal
+        score = 0.0
+        total_weight = 0.0
+
+        if self.metric_config['text_density']:
+            density = text_len / tag_len if tag_len > 0 else 0
+            score += self.metric_weights['text_density'] * density
+            total_weight += self.metric_weights['text_density']
+
+        if self.metric_config['link_density']:
+            density = 1 - (link_text_len / text_len if text_len > 0 else 0)
+            score += self.metric_weights['link_density'] * density
+            total_weight += self.metric_weights['link_density']
+
+        if self.metric_config['tag_weight']:
+            tag_score = self.tag_weights.get(metrics['tag_name'], 0.5)
+            score += self.metric_weights['tag_weight'] * tag_score
+            total_weight += self.metric_weights['tag_weight']
+
+        if self.metric_config['class_id_weight']:
+            class_score = self._compute_class_id_weight(metrics['node'])
+            score += self.metric_weights['class_id_weight'] * max(0, class_score)
+            total_weight += self.metric_weights['class_id_weight']
+
+        if self.metric_config['text_length']:
+            score += self.metric_weights['text_length'] * math.log(text_len + 1)
+            total_weight += self.metric_weights['text_length']
+
+        return score / total_weight if total_weight > 0 else 0
+
+    def _compute_class_id_weight(self, node):
+        """Computes the class ID weight"""
+        class_id_score = 0
+        if 'class' in node.attrs:
+            classes = ' '.join(node['class'])
+            if self.negative_patterns.match(classes):
+                class_id_score -= 0.5
+        if 'id' in node.attrs:
+            element_id = node['id']
+            if self.negative_patterns.match(element_id):
+                class_id_score -= 0.5
+        return class_id_score

crawl4ai/content_scraping_strategy.py

@@ -0,0 +1,723 @@
+import re  # Point 1: Pre-Compile Regular Expressions
+import time
+from abc import ABC, abstractmethod
+from typing import Dict, Any, Optional
+from bs4 import BeautifulSoup
+from concurrent.futures import ThreadPoolExecutor
+import asyncio, requests, re, os
+from .config import *
+from bs4 import element, NavigableString, Comment
+from bs4 import PageElement, Tag
+from urllib.parse import urljoin
+from requests.exceptions import InvalidSchema
+# from .content_cleaning_strategy import ContentCleaningStrategy
+from .content_filter_strategy import RelevantContentFilter, BM25ContentFilter#, HeuristicContentFilter
+from .markdown_generation_strategy import MarkdownGenerationStrategy, DefaultMarkdownGenerator
+from .models import MarkdownGenerationResult
+from .utils import (
+    extract_metadata,
+    normalize_url,
+    is_external_url,    
+    get_base_domain,    
+)
+
+
+# Pre-compile regular expressions for Open Graph and Twitter metadata
+OG_REGEX = re.compile(r'^og:')
+TWITTER_REGEX = re.compile(r'^twitter:')
+DIMENSION_REGEX = re.compile(r"(\d+)(\D*)")
+
+# Function to parse image height/width value and units
+def parse_dimension(dimension):
+    if dimension:
+        # match = re.match(r"(\d+)(\D*)", dimension)
+        match = DIMENSION_REGEX.match(dimension)
+        if match:
+            number = int(match.group(1))
+            unit = match.group(2) or 'px'  # Default unit is 'px' if not specified
+            return number, unit
+    return None, None
+
+# Fetch image file metadata to extract size and extension
+def fetch_image_file_size(img, base_url):
+    #If src is relative path construct full URL, if not it may be CDN URL
+    img_url = urljoin(base_url,img.get('src'))
+    try:
+        response = requests.head(img_url)
+        if response.status_code == 200:
+            return response.headers.get('Content-Length',None)
+        else:
+            print(f"Failed to retrieve file size for {img_url}")
+            return None
+    except InvalidSchema as e:
+        return None
+    finally:
+        return
+
+class ContentScrapingStrategy(ABC):
+    @abstractmethod
+    def scrap(self, url: str, html: str, **kwargs) -> Dict[str, Any]:
+        pass
+
+    @abstractmethod
+    async def ascrap(self, url: str, html: str, **kwargs) -> Dict[str, Any]:
+        pass
+
+class WebScrapingStrategy(ContentScrapingStrategy):
+    """
+    Class for web content scraping. Perhaps the most important class. 
+    
+    How it works:
+    1. Extract content from HTML using BeautifulSoup.
+    2. Clean the extracted content using a content cleaning strategy.
+    3. Filter the cleaned content using a content filtering strategy.
+    4. Generate markdown content from the filtered content.
+    5. Return the markdown content.
+    """
+    
+    def __init__(self, logger=None):
+        self.logger = logger
+
+    def _log(self, level, message, tag="SCRAPE", **kwargs):
+        """Helper method to safely use logger."""
+        if self.logger:
+            log_method = getattr(self.logger, level)
+            log_method(message=message, tag=tag, **kwargs)
+                
+    def scrap(self, url: str, html: str, **kwargs) -> Dict[str, Any]:
+        """
+        Main entry point for content scraping.  
+
+        Args:
+            url (str): The URL of the page to scrape.
+            html (str): The HTML content of the page.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the scraped content. This dictionary contains the following keys:
+
+            - 'markdown': The generated markdown content, type is str, however soon will become MarkdownGenerationResult via 'markdown.raw_markdown'.
+            - 'fit_markdown': The generated markdown content with relevant content filtered, this will be removed soon and available in 'markdown.fit_markdown'.
+            - 'fit_html': The HTML content with relevant content filtered, this will be removed soon and available in 'markdown.fit_html'.
+            - 'markdown_v2': The generated markdown content with relevant content filtered, this is temporary and will be removed soon and replaced with 'markdown'
+        """
+        return self._scrap(url, html, is_async=False, **kwargs)
+
+    async def ascrap(self, url: str, html: str, **kwargs) -> Dict[str, Any]:
+        """
+        Main entry point for asynchronous content scraping.
+
+        Args:
+            url (str): The URL of the page to scrape.
+            html (str): The HTML content of the page.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the scraped content. This dictionary contains the following keys:
+
+            - 'markdown': The generated markdown content, type is str, however soon will become MarkdownGenerationResult via 'markdown.raw_markdown'.
+            - 'fit_markdown': The generated markdown content with relevant content filtered, this will be removed soon and available in 'markdown.fit_markdown'.
+            - 'fit_html': The HTML content with relevant content filtered, this will be removed soon and available in 'markdown.fit_html'.
+            - 'markdown_v2': The generated markdown content with relevant content filtered, this is temporary and will be removed soon and replaced with 'markdown'
+        """
+        return await asyncio.to_thread(self._scrap, url, html, **kwargs)
+
+    def flatten_nested_elements(self, node):
+        """
+        Flatten nested elements in a HTML tree.
+
+        Args:
+            node (Tag): The root node of the HTML tree.
+
+        Returns:
+            Tag: The flattened HTML tree.
+        """
+        if isinstance(node, NavigableString):
+            return node
+        if len(node.contents) == 1 and isinstance(node.contents[0], Tag) and node.contents[0].name == node.name:
+            return self.flatten_nested_elements(node.contents[0])
+        node.contents = [self.flatten_nested_elements(child) for child in node.contents]
+        return node
+
+    def find_closest_parent_with_useful_text(self, tag, **kwargs):
+        """
+        Find the closest parent with useful text.
+
+        Args:
+            tag (Tag): The starting tag to search from.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            Tag: The closest parent with useful text, or None if not found.
+        """
+        image_description_min_word_threshold = kwargs.get('image_description_min_word_threshold', IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD)
+        current_tag = tag
+        while current_tag:
+            current_tag = current_tag.parent
+            # Get the text content of the parent tag
+            if current_tag:
+                text_content = current_tag.get_text(separator=' ',strip=True)
+                # Check if the text content has at least word_count_threshold
+                if len(text_content.split()) >= image_description_min_word_threshold:
+                    return text_content
+        return None
+
+    def remove_unwanted_attributes(self, element, important_attrs, keep_data_attributes=False):
+        """
+        Remove unwanted attributes from an HTML element.
+
+        Args:    
+            element (Tag): The HTML element to remove attributes from.
+            important_attrs (list): List of important attributes to keep.
+            keep_data_attributes (bool): Whether to keep data attributes.
+
+        Returns:
+            None
+        """
+        attrs_to_remove = []
+        for attr in element.attrs:
+            if attr not in important_attrs:
+                if keep_data_attributes:
+                    if not attr.startswith('data-'):
+                        attrs_to_remove.append(attr)
+                else:
+                    attrs_to_remove.append(attr)
+        
+        for attr in attrs_to_remove:
+            del element[attr]
+
+    def process_image(self, img, url, index, total_images, **kwargs):
+        """
+        Process an image element.
+        
+        How it works:
+        1. Check if the image has valid display and inside undesired html elements.
+        2. Score an image for it's usefulness.
+        3. Extract image file metadata to extract size and extension.
+        4. Generate a dictionary with the processed image information.
+        5. Return the processed image information.
+
+        Args:
+            img (Tag): The image element to process.
+            url (str): The URL of the page containing the image.
+            index (int): The index of the image in the list of images.
+            total_images (int): The total number of images in the list.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            dict: A dictionary containing the processed image information.
+        """
+        parse_srcset = lambda s: [{'url': u.strip().split()[0], 'width': u.strip().split()[-1].rstrip('w') 
+                        if ' ' in u else None} 
+                        for u in [f"http{p}" for p in s.split("http") if p]]
+        
+        # Constants for checks
+        classes_to_check = frozenset(['button', 'icon', 'logo'])
+        tags_to_check = frozenset(['button', 'input'])
+        image_formats = frozenset(['jpg', 'jpeg', 'png', 'webp', 'avif', 'gif'])
+        
+        # Pre-fetch commonly used attributes
+        style = img.get('style', '')
+        alt = img.get('alt', '')
+        src = img.get('src', '')
+        data_src = img.get('data-src', '')
+        srcset = img.get('srcset', '')
+        data_srcset = img.get('data-srcset', '')        
+        width = img.get('width')
+        height = img.get('height')
+        parent = img.parent
+        parent_classes = parent.get('class', [])
+
+        # Quick validation checks
+        if ('display:none' in style or
+            parent.name in tags_to_check or
+            any(c in cls for c in parent_classes for cls in classes_to_check) or
+            any(c in src for c in classes_to_check) or
+            any(c in alt for c in classes_to_check)):
+            return None
+
+        # Quick score calculation
+        score = 0
+        if width and width.isdigit():
+            width_val = int(width)
+            score += 1 if width_val > 150 else 0
+        if height and height.isdigit():
+            height_val = int(height)
+            score += 1 if height_val > 150 else 0
+        if alt:
+            score += 1
+        score += index/total_images < 0.5
+        
+        # image_format = ''
+        # if "data:image/" in src:
+        #     image_format = src.split(',')[0].split(';')[0].split('/')[1].split(';')[0]
+        # else:
+        #     image_format = os.path.splitext(src)[1].lower().strip('.').split('?')[0]
+        
+        # if image_format in ('jpg', 'png', 'webp', 'avif'):
+        #     score += 1
+            
+            
+        # Check for image format in all possible sources
+        def has_image_format(url):
+            return any(fmt in url.lower() for fmt in image_formats)
+        
+        # Score for having proper image sources
+        if any(has_image_format(url) for url in [src, data_src, srcset, data_srcset]):
+            score += 1
+        if srcset or data_srcset:
+            score += 1
+        if img.find_parent('picture'):
+            score += 1
+        
+        # Detect format from any available source
+        detected_format = None
+        for url in [src, data_src, srcset, data_srcset]:
+            if url:
+                format_matches = [fmt for fmt in image_formats if fmt in url.lower()]
+                if format_matches:
+                    detected_format = format_matches[0]
+                    break            
+
+        if score <= kwargs.get('image_score_threshold', IMAGE_SCORE_THRESHOLD):
+            return None
+
+        # Use set for deduplication
+        unique_urls = set()
+        image_variants = []
+        
+        # Generate a unique group ID for this set of variants
+        group_id = index 
+        
+        # Base image info template
+        image_description_min_word_threshold = kwargs.get('image_description_min_word_threshold', IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD)
+        base_info = {
+            'alt': alt,
+            'desc': self.find_closest_parent_with_useful_text(img, **kwargs),
+            'score': score,
+            'type': 'image',
+            'group_id': group_id, # Group ID for this set of variants
+            'format': detected_format,
+        }
+
+        # Inline function for adding variants
+        def add_variant(src, width=None):
+            if src and not src.startswith('data:') and src not in unique_urls:
+                unique_urls.add(src)
+                image_variants.append({**base_info, 'src': src, 'width': width})
+
+        # Process all sources
+        add_variant(src)
+        add_variant(data_src)
+        
+        # Handle srcset and data-srcset in one pass
+        for attr in ('srcset', 'data-srcset'):
+            if value := img.get(attr):
+                for source in parse_srcset(value):
+                    add_variant(source['url'], source['width'])
+
+        # Quick picture element check
+        if picture := img.find_parent('picture'):
+            for source in picture.find_all('source'):
+                if srcset := source.get('srcset'):
+                    for src in parse_srcset(srcset):
+                        add_variant(src['url'], src['width'])
+
+        # Framework-specific attributes in one pass
+        for attr, value in img.attrs.items():
+            if attr.startswith('data-') and ('src' in attr or 'srcset' in attr) and 'http' in value:
+                add_variant(value)
+
+        return image_variants if image_variants else None
+
+    def process_element(self, url, element: PageElement, **kwargs) -> Dict[str, Any]:        
+        """
+        Process an HTML element.
+        
+        How it works:
+        1. Check if the element is an image, video, or audio.
+        2. Extract the element's attributes and content.
+        3. Process the element based on its type.
+        4. Return the processed element information.
+
+        Args:
+            url (str): The URL of the page containing the element.
+            element (Tag): The HTML element to process.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            dict: A dictionary containing the processed element information.
+        """
+        media = {'images': [], 'videos': [], 'audios': []}
+        internal_links_dict = {}
+        external_links_dict = {}
+        self._process_element(
+            url,
+            element,
+            media,
+            internal_links_dict,
+            external_links_dict,
+            **kwargs
+        )
+        return {
+            'media': media,
+            'internal_links_dict': internal_links_dict,
+            'external_links_dict': external_links_dict
+        }
+        
+    def _process_element(self, url, element: PageElement,  media: Dict[str, Any], internal_links_dict: Dict[str, Any], external_links_dict: Dict[str, Any], **kwargs) -> bool:
+        """
+        Process an HTML element.        
+        """
+        try:
+            if isinstance(element, NavigableString):
+                if isinstance(element, Comment):
+                    element.extract()
+                return False
+            
+            # if element.name == 'img':
+            #     process_image(element, url, 0, 1)
+            #     return True
+            base_domain = kwargs.get("base_domain", get_base_domain(url))
+
+            if element.name in ['script', 'style', 'link', 'meta', 'noscript']:
+                element.decompose()
+                return False
+
+            keep_element = False
+            
+            exclude_domains = kwargs.get('exclude_domains', [])
+            # exclude_social_media_domains = kwargs.get('exclude_social_media_domains', set(SOCIAL_MEDIA_DOMAINS))
+            # exclude_social_media_domains = SOCIAL_MEDIA_DOMAINS + kwargs.get('exclude_social_media_domains', [])
+            # exclude_social_media_domains = list(set(exclude_social_media_domains))
+            
+            try:
+                if element.name == 'a' and element.get('href'):
+                    href = element.get('href', '').strip()
+                    if not href:  # Skip empty hrefs
+                        return False
+                        
+                    url_base = url.split('/')[2]
+                    
+                    # Normalize the URL
+                    try:
+                        normalized_href = normalize_url(href, url)
+                    except ValueError as e:
+                        # logging.warning(f"Invalid URL format: {href}, Error: {str(e)}")
+                        return False
+                        
+                    link_data = {
+                        'href': normalized_href,
+                        'text': element.get_text().strip(),
+                        'title': element.get('title', '').strip(),
+                        'base_domain': base_domain
+                    }
+                                        
+                    is_external = is_external_url(normalized_href, base_domain)
+                            
+                    keep_element = True
+                    
+                    # Handle external link exclusions
+                    if is_external:
+                        link_base_domain = get_base_domain(normalized_href)
+                        link_data['base_domain'] = link_base_domain
+                        if kwargs.get('exclude_external_links', False):
+                            element.decompose()
+                            return False
+                        # elif kwargs.get('exclude_social_media_links', False):
+                        #     if link_base_domain in exclude_social_media_domains:
+                        #         element.decompose()
+                        #         return False
+                            # if any(domain in normalized_href.lower() for domain in exclude_social_media_domains):
+                            #     element.decompose()
+                            #     return False
+                        elif exclude_domains:
+                            if link_base_domain in exclude_domains:
+                                element.decompose()
+                                return False
+                            # if any(domain in normalized_href.lower() for domain in kwargs.get('exclude_domains', [])):
+                            #     element.decompose()
+                            #     return False
+
+                    if is_external:
+                        if normalized_href not in external_links_dict:
+                            external_links_dict[normalized_href] = link_data
+                    else:
+                        if normalized_href not in internal_links_dict:
+                            internal_links_dict[normalized_href] = link_data
+
+                                
+            except Exception as e:
+                raise Exception(f"Error processing links: {str(e)}")
+
+            try:
+                if element.name == 'img':
+                    potential_sources = ['src', 'data-src', 'srcset' 'data-lazy-src', 'data-original']
+                    src = element.get('src', '')
+                    while not src and potential_sources:
+                        src = element.get(potential_sources.pop(0), '')
+                    if not src:
+                        element.decompose()
+                        return False
+                    
+                    # If it is srcset pick up the first image
+                    if 'srcset' in element.attrs:
+                        src = element.attrs['srcset'].split(',')[0].split(' ')[0]
+                        
+                    # If image src is internal, then skip
+                    if not is_external_url(src, base_domain):
+                        return True
+                    
+                    image_src_base_domain = get_base_domain(src)
+                    
+                    # Check flag if we should remove external images
+                    if kwargs.get('exclude_external_images', False):
+                        element.decompose()
+                        return False
+                        # src_url_base = src.split('/')[2]
+                        # url_base = url.split('/')[2]
+                        # if url_base not in src_url_base:
+                        #     element.decompose()
+                        #     return False
+                        
+                    # if kwargs.get('exclude_social_media_links', False):
+                    #     if image_src_base_domain in exclude_social_media_domains:
+                    #         element.decompose()
+                    #         return False
+                        # src_url_base = src.split('/')[2]
+                        # url_base = url.split('/')[2]
+                        # if any(domain in src for domain in exclude_social_media_domains):
+                        #     element.decompose()
+                        #     return False
+                        
+                    # Handle exclude domains
+                    if exclude_domains:                        
+                        if image_src_base_domain in exclude_domains:
+                            element.decompose()
+                            return False
+                        # if any(domain in src for domain in kwargs.get('exclude_domains', [])):
+                        #     element.decompose()
+                        #     return False
+                    
+                    return True  # Always keep image elements
+            except Exception as e:
+                raise "Error processing images"
+            
+            
+            # Check if flag to remove all forms is set
+            if kwargs.get('remove_forms', False) and element.name == 'form':
+                element.decompose()
+                return False
+            
+            if element.name in ['video', 'audio']:
+                media[f"{element.name}s"].append({
+                    'src': element.get('src'),
+                    'alt': element.get('alt'),
+                    'type': element.name,
+                    'description': self.find_closest_parent_with_useful_text(element, **kwargs)
+                })
+                source_tags = element.find_all('source')
+                for source_tag in source_tags:
+                    media[f"{element.name}s"].append({
+                    'src': source_tag.get('src'),
+                    'alt': element.get('alt'),
+                    'type': element.name,
+                    'description': self.find_closest_parent_with_useful_text(element, **kwargs)
+                })
+                return True  # Always keep video and audio elements
+
+            if element.name in ONLY_TEXT_ELIGIBLE_TAGS:
+                if kwargs.get('only_text', False):
+                    element.replace_with(element.get_text())
+
+            try:
+                self.remove_unwanted_attributes(element, IMPORTANT_ATTRS, kwargs.get('keep_data_attributes', False))
+            except Exception as e:
+                # print('Error removing unwanted attributes:', str(e))
+                self._log('error',
+                    message="Error removing unwanted attributes: {error}",
+                    tag="SCRAPE",
+                    params={"error": str(e)}
+                )
+            # Process children
+            for child in list(element.children):
+                if isinstance(child, NavigableString) and not isinstance(child, Comment):
+                    if len(child.strip()) > 0:
+                        keep_element = True
+                else:
+                    if self._process_element(url, child, media, internal_links_dict, external_links_dict, **kwargs):
+                        keep_element = True
+                
+
+            # Check word count
+            word_count_threshold = kwargs.get('word_count_threshold', MIN_WORD_THRESHOLD)
+            if not keep_element:
+                word_count = len(element.get_text(strip=True).split())
+                keep_element = word_count >= word_count_threshold
+
+            if not keep_element:
+                element.decompose()
+
+            return keep_element
+        except Exception as e:
+            # print('Error processing element:', str(e))
+            self._log('error',
+                message="Error processing element: {error}",
+                tag="SCRAPE",
+                params={"error": str(e)}
+            )                
+            return False
+
+    def _scrap(self, url: str, html: str, word_count_threshold: int = MIN_WORD_THRESHOLD, css_selector: str = None, **kwargs) -> Dict[str, Any]:
+        """
+        Extract content from HTML using BeautifulSoup.
+
+        Args:
+            url (str): The URL of the page to scrape.
+            html (str): The HTML content of the page to scrape.
+            word_count_threshold (int): The minimum word count threshold for content extraction.
+            css_selector (str): The CSS selector to use for content extraction.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            dict: A dictionary containing the extracted content.
+        """
+        success = True
+        if not html:
+            return None
+
+        parser_type = kwargs.get('parser', 'lxml')
+        soup = BeautifulSoup(html, parser_type)
+        body = soup.body
+        base_domain = get_base_domain(url)
+        
+        try:
+            meta = extract_metadata("", soup)
+        except Exception as e:
+            self._log('error', 
+                message="Error extracting metadata: {error}",
+                tag="SCRAPE",
+                params={"error": str(e)}
+            )            
+            meta = {}
+        
+        # Handle tag-based removal first - faster than CSS selection
+        excluded_tags = set(kwargs.get('excluded_tags', []) or [])  
+        if excluded_tags:
+            for element in body.find_all(lambda tag: tag.name in excluded_tags):
+                element.extract()
+        
+        # Handle CSS selector-based removal
+        excluded_selector = kwargs.get('excluded_selector', '')
+        if excluded_selector:
+            is_single_selector = ',' not in excluded_selector and ' ' not in excluded_selector
+            if is_single_selector:
+                while element := body.select_one(excluded_selector):
+                    element.extract()
+            else:
+                for element in body.select(excluded_selector):
+                    element.extract()  
+        
+        if css_selector:
+            selected_elements = body.select(css_selector)
+            if not selected_elements:
+                return {
+                    'markdown': '',
+                    'cleaned_html': '',
+                    'success': True,
+                    'media': {'images': [], 'videos': [], 'audios': []},
+                    'links': {'internal': [], 'external': []},
+                    'metadata': {},
+                    'message': f"No elements found for CSS selector: {css_selector}"
+                }
+                # raise InvalidCSSSelectorError(f"Invalid CSS selector, No elements found for CSS selector: {css_selector}")
+            body = soup.new_tag('div')
+            for el in selected_elements:
+                body.append(el)
+
+        kwargs['exclude_social_media_domains'] = set(kwargs.get('exclude_social_media_domains', []) + SOCIAL_MEDIA_DOMAINS)
+        kwargs['exclude_domains'] = set(kwargs.get('exclude_domains', []))
+        if kwargs.get('exclude_social_media_links', False):
+            kwargs['exclude_domains'] = kwargs['exclude_domains'].union(kwargs['exclude_social_media_domains'])
+        
+        result_obj = self.process_element(
+            url, 
+            body, 
+            word_count_threshold = word_count_threshold, 
+            base_domain=base_domain,
+            **kwargs
+        )
+        
+        links = {'internal': [], 'external': []}
+        media = result_obj['media']
+        internal_links_dict = result_obj['internal_links_dict']
+        external_links_dict = result_obj['external_links_dict']
+        
+        # Update the links dictionary with unique links
+        links['internal'] = list(internal_links_dict.values())
+        links['external'] = list(external_links_dict.values())
+
+        # # Process images using ThreadPoolExecutor
+        imgs = body.find_all('img')
+        
+        media['images'] = [
+            img for result in (self.process_image(img, url, i, len(imgs)) 
+                            for i, img in enumerate(imgs))
+            if result is not None
+            for img in result
+        ]
+
+        body = self.flatten_nested_elements(body)
+        base64_pattern = re.compile(r'data:image/[^;]+;base64,([^"]+)')
+        for img in imgs:
+            src = img.get('src', '')
+            if base64_pattern.match(src):
+                # Replace base64 data with empty string
+                img['src'] = base64_pattern.sub('', src)
+                
+        str_body = ""
+        try:
+            str_body = body.encode_contents().decode('utf-8')
+        except Exception as e:
+            # Reset body to the original HTML
+            success = False
+            body = BeautifulSoup(html, 'html.parser')
+            
+            # Create a new div with a special ID
+            error_div = body.new_tag('div', id='crawl4ai_error_message')
+            error_div.string = '''
+            Crawl4AI Error: This page is not fully supported.
+            
+            Possible reasons:
+            1. The page may have restrictions that prevent crawling.
+            2. The page might not be fully loaded.
+            
+            Suggestions:
+            - Try calling the crawl function with these parameters:
+            magic=True,
+            - Set headless=False to visualize what's happening on the page.
+            
+            If the issue persists, please check the page's structure and any potential anti-crawling measures.
+            '''
+            
+            # Append the error div to the body
+            body.body.append(error_div)
+            str_body = body.encode_contents().decode('utf-8')
+            
+            print(f"[LOG] 😧 Error: After processing the crawled HTML and removing irrelevant tags, nothing was left in the page. Check the markdown for further details.")
+            self._log('error',
+                message="After processing the crawled HTML and removing irrelevant tags, nothing was left in the page. Check the markdown for further details.",
+                tag="SCRAPE"
+            )
+
+        cleaned_html = str_body.replace('\n\n', '\n').replace('  ', ' ')
+
+        
+        return {
+            # **markdown_content,
+            'cleaned_html': cleaned_html,
+            'success': success,
+            'media': media,
+            'links': links,
+            'metadata': meta
+        }

crawl4ai/crawler_strategy.py

@@ -0,0 +1,360 @@
+from abc import ABC, abstractmethod
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.chrome.options import Options
+from selenium.common.exceptions import InvalidArgumentException, WebDriverException
+# from selenium.webdriver.chrome.service import Service as ChromeService
+# from webdriver_manager.chrome import ChromeDriverManager
+# from urllib3.exceptions import MaxRetryError
+
+from .config import *
+import logging, time
+import base64
+from PIL import Image, ImageDraw, ImageFont
+from io import BytesIO
+from typing import List, Callable
+import requests
+import os
+from pathlib import Path
+from .utils import *
+
+logger = logging.getLogger('selenium.webdriver.remote.remote_connection')
+logger.setLevel(logging.WARNING)
+
+logger_driver = logging.getLogger('selenium.webdriver.common.service')
+logger_driver.setLevel(logging.WARNING)
+
+urllib3_logger = logging.getLogger('urllib3.connectionpool')
+urllib3_logger.setLevel(logging.WARNING)
+
+# Disable http.client logging
+http_client_logger = logging.getLogger('http.client')
+http_client_logger.setLevel(logging.WARNING)
+
+# Disable driver_finder and service logging
+driver_finder_logger = logging.getLogger('selenium.webdriver.common.driver_finder')
+driver_finder_logger.setLevel(logging.WARNING)
+
+
+
+
+class CrawlerStrategy(ABC):
+    @abstractmethod
+    def crawl(self, url: str, **kwargs) -> str:
+        pass
+    
+    @abstractmethod
+    def take_screenshot(self, save_path: str):
+        pass
+    
+    @abstractmethod
+    def update_user_agent(self, user_agent: str):
+        pass
+    
+    @abstractmethod
+    def set_hook(self, hook_type: str, hook: Callable):
+        pass
+
+class CloudCrawlerStrategy(CrawlerStrategy):
+    def __init__(self, use_cached_html = False):
+        super().__init__()
+        self.use_cached_html = use_cached_html
+        
+    def crawl(self, url: str) -> str:
+        data = {
+            "urls": [url],
+            "include_raw_html": True,
+            "forced": True,
+            "extract_blocks": False,
+        }
+
+        response = requests.post("http://crawl4ai.uccode.io/crawl", json=data)
+        response = response.json()
+        html = response["results"][0]["html"]
+        return sanitize_input_encode(html)
+
+class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
+    def __init__(self, use_cached_html=False, js_code=None, **kwargs):
+        super().__init__()
+        print("[LOG] 🚀 Initializing LocalSeleniumCrawlerStrategy")
+        self.options = Options()
+        self.options.headless = True
+        if kwargs.get("proxy"):
+            self.options.add_argument("--proxy-server={}".format(kwargs.get("proxy")))
+        if kwargs.get("user_agent"):
+            self.options.add_argument("--user-agent=" + kwargs.get("user_agent"))
+        else:
+            user_agent = kwargs.get("user_agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
+            self.options.add_argument(f"--user-agent={user_agent}")
+            self.options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
+                  
+        self.options.headless = kwargs.get("headless", True)
+        if self.options.headless:
+            self.options.add_argument("--headless")
+        
+        self.options.add_argument("--disable-gpu")  
+        self.options.add_argument("--window-size=1920,1080")
+        self.options.add_argument("--no-sandbox")
+        self.options.add_argument("--disable-dev-shm-usage")
+        self.options.add_argument("--disable-blink-features=AutomationControlled")     
+        
+        # self.options.add_argument("--disable-dev-shm-usage")
+        self.options.add_argument("--disable-gpu")
+        # self.options.add_argument("--disable-extensions")
+        # self.options.add_argument("--disable-infobars")
+        # self.options.add_argument("--disable-logging")
+        # self.options.add_argument("--disable-popup-blocking")
+        # self.options.add_argument("--disable-translate")
+        # self.options.add_argument("--disable-default-apps")
+        # self.options.add_argument("--disable-background-networking")
+        # self.options.add_argument("--disable-sync")
+        # self.options.add_argument("--disable-features=NetworkService,NetworkServiceInProcess")
+        # self.options.add_argument("--disable-browser-side-navigation")
+        # self.options.add_argument("--dns-prefetch-disable")
+        # self.options.add_argument("--disable-web-security")
+        self.options.add_argument("--log-level=3")
+        self.use_cached_html = use_cached_html
+        self.use_cached_html = use_cached_html
+        self.js_code = js_code
+        self.verbose = kwargs.get("verbose", False)
+        
+        # Hooks
+        self.hooks = {
+            'on_driver_created': None,
+            'on_user_agent_updated': None,
+            'before_get_url': None,
+            'after_get_url': None,
+            'before_return_html': None
+        }
+
+        # chromedriver_autoinstaller.install()
+        # import chromedriver_autoinstaller
+        # crawl4ai_folder = os.path.join(os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai")
+        # driver = webdriver.Chrome(service=ChromeService(ChromeDriverManager().install()), options=self.options)
+        # chromedriver_path = chromedriver_autoinstaller.install()
+        # chromedriver_path = chromedriver_autoinstaller.utils.download_chromedriver()
+        # self.service = Service(chromedriver_autoinstaller.install())
+        
+        
+        # chromedriver_path = ChromeDriverManager().install()
+        # self.service = Service(chromedriver_path)
+        # self.service.log_path = "NUL"
+        # self.driver = webdriver.Chrome(service=self.service, options=self.options)
+        
+        # Use selenium-manager (built into Selenium 4.10.0+)
+        self.service = Service()
+        self.driver = webdriver.Chrome(options=self.options)
+        
+        self.driver = self.execute_hook('on_driver_created', self.driver)
+        
+        if kwargs.get("cookies"):
+            for cookie in kwargs.get("cookies"):
+                self.driver.add_cookie(cookie)
+            
+        
+
+    def set_hook(self, hook_type: str, hook: Callable):
+        if hook_type in self.hooks:
+            self.hooks[hook_type] = hook
+        else:
+            raise ValueError(f"Invalid hook type: {hook_type}")
+    
+    def execute_hook(self, hook_type: str, *args):
+        hook = self.hooks.get(hook_type)
+        if hook:
+            result = hook(*args)
+            if result is not None:
+                if isinstance(result, webdriver.Chrome):
+                    return result
+                else:
+                    raise TypeError(f"Hook {hook_type} must return an instance of webdriver.Chrome or None.")
+        # If the hook returns None or there is no hook, return self.driver
+        return self.driver
+
+    def update_user_agent(self, user_agent: str):
+        self.options.add_argument(f"user-agent={user_agent}")
+        self.driver.quit()
+        self.driver = webdriver.Chrome(service=self.service, options=self.options)
+        self.driver = self.execute_hook('on_user_agent_updated', self.driver)
+
+    def set_custom_headers(self, headers: dict):
+        # Enable Network domain for sending headers
+        self.driver.execute_cdp_cmd('Network.enable', {})
+        # Set extra HTTP headers
+        self.driver.execute_cdp_cmd('Network.setExtraHTTPHeaders', {'headers': headers})
+
+    def _ensure_page_load(self,  max_checks=6, check_interval=0.01):
+        initial_length = len(self.driver.page_source)
+        
+        for ix in range(max_checks):
+            # print(f"Checking page load: {ix}")
+            time.sleep(check_interval)
+            current_length = len(self.driver.page_source)
+            
+            if current_length != initial_length:
+                break
+
+        return self.driver.page_source
+    
+    def crawl(self, url: str, **kwargs) -> str:
+        # Create md5 hash of the URL
+        import hashlib
+        url_hash = hashlib.md5(url.encode()).hexdigest()
+        
+        if self.use_cached_html:
+            cache_file_path = os.path.join(os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai", "cache", url_hash)
+            if os.path.exists(cache_file_path):
+                with open(cache_file_path, "r") as f:
+                    return sanitize_input_encode(f.read())
+
+        try:
+            self.driver = self.execute_hook('before_get_url', self.driver)
+            if self.verbose:
+                print(f"[LOG] 🕸️ Crawling {url} using LocalSeleniumCrawlerStrategy...")
+            self.driver.get(url) #<html><head></head><body></body></html>
+            
+            WebDriverWait(self.driver, 20).until(
+                lambda d: d.execute_script('return document.readyState') == 'complete'
+            )
+            WebDriverWait(self.driver, 10).until(
+                EC.presence_of_all_elements_located((By.TAG_NAME, "body"))
+            )
+            
+            self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
+            
+            self.driver = self.execute_hook('after_get_url', self.driver)
+            html = sanitize_input_encode(self._ensure_page_load()) # self.driver.page_source                                        
+            can_not_be_done_headless = False # Look at my creativity for naming variables
+            
+            # TODO: Very ugly approach, but promise to change it!
+            if kwargs.get('bypass_headless', False) or html == "<html><head></head><body></body></html>":
+                print("[LOG] 🙌 Page could not be loaded in headless mode. Trying non-headless mode...")
+                can_not_be_done_headless = True
+                options = Options()
+                options.headless = False
+                # set window size very small
+                options.add_argument("--window-size=5,5")
+                driver = webdriver.Chrome(service=self.service, options=options)
+                driver.get(url)
+                self.driver = self.execute_hook('after_get_url', driver)
+                html = sanitize_input_encode(driver.page_source)
+                driver.quit()
+            
+            # Execute JS code if provided
+            self.js_code = kwargs.get("js_code", self.js_code)
+            if self.js_code and type(self.js_code) == str:
+                self.driver.execute_script(self.js_code)
+                # Optionally, wait for some condition after executing the JS code
+                WebDriverWait(self.driver, 10).until(
+                    lambda driver: driver.execute_script("return document.readyState") == "complete"
+                )
+            elif self.js_code and type(self.js_code) == list:
+                for js in self.js_code:
+                    self.driver.execute_script(js)
+                    WebDriverWait(self.driver, 10).until(
+                        lambda driver: driver.execute_script("return document.readyState") == "complete"
+                    )
+            
+            # Optionally, wait for some condition after executing the JS code : Contributed by (https://github.com/jonymusky)
+            wait_for = kwargs.get('wait_for', False)
+            if wait_for:
+                if callable(wait_for):
+                    print("[LOG] 🔄 Waiting for condition...")
+                    WebDriverWait(self.driver, 20).until(wait_for)
+                else:
+                    print("[LOG] 🔄 Waiting for condition...")
+                    WebDriverWait(self.driver, 20).until(
+                        EC.presence_of_element_located((By.CSS_SELECTOR, wait_for))
+                    ) 
+            
+            if not can_not_be_done_headless:
+                html = sanitize_input_encode(self.driver.page_source)
+            self.driver = self.execute_hook('before_return_html', self.driver, html)
+            
+            # Store in cache
+            cache_file_path = os.path.join(os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai", "cache", url_hash)
+            with open(cache_file_path, "w", encoding="utf-8") as f:
+                f.write(html)
+                
+            if self.verbose:
+                print(f"[LOG] ✅ Crawled {url} successfully!")
+            
+            return html
+        except InvalidArgumentException as e:
+            if not hasattr(e, 'msg'):
+                e.msg = sanitize_input_encode(str(e))
+            raise InvalidArgumentException(f"Failed to crawl {url}: {e.msg}")
+        except WebDriverException as e:
+            # If e does nlt have msg attribute create it and set it to str(e)
+            if not hasattr(e, 'msg'):
+                e.msg = sanitize_input_encode(str(e))
+            raise WebDriverException(f"Failed to crawl {url}: {e.msg}")  
+        except Exception as e:
+            if not hasattr(e, 'msg'):
+                e.msg = sanitize_input_encode(str(e))
+            raise Exception(f"Failed to crawl {url}: {e.msg}")
+
+    def take_screenshot(self) -> str:
+        try:
+            # Get the dimensions of the page
+            total_width = self.driver.execute_script("return document.body.scrollWidth")
+            total_height = self.driver.execute_script("return document.body.scrollHeight")
+
+            # Set the window size to the dimensions of the page
+            self.driver.set_window_size(total_width, total_height)
+
+            # Take screenshot
+            screenshot = self.driver.get_screenshot_as_png()
+
+            # Open the screenshot with PIL
+            image = Image.open(BytesIO(screenshot))
+
+            # Convert image to RGB mode (this will handle both RGB and RGBA images)
+            rgb_image = image.convert('RGB')
+
+            # Convert to JPEG and compress
+            buffered = BytesIO()
+            rgb_image.save(buffered, format="JPEG", quality=85)
+            img_base64 = base64.b64encode(buffered.getvalue()).decode('utf-8')
+
+            if self.verbose:
+                print(f"[LOG] 📸 Screenshot taken and converted to base64")
+
+            return img_base64
+        except Exception as e:
+            error_message = sanitize_input_encode(f"Failed to take screenshot: {str(e)}")
+            print(error_message)
+
+            # Generate an image with black background
+            img = Image.new('RGB', (800, 600), color='black')
+            draw = ImageDraw.Draw(img)
+            
+            # Load a font
+            try:
+                font = ImageFont.truetype("arial.ttf", 40)
+            except IOError:
+                font = ImageFont.load_default()
+
+            # Define text color and wrap the text
+            text_color = (255, 255, 255)
+            max_width = 780
+            wrapped_text = wrap_text(draw, error_message, font, max_width)
+
+            # Calculate text position
+            text_position = (10, 10)
+            
+            # Draw the text on the image
+            draw.text(text_position, wrapped_text, fill=text_color, font=font)
+            
+            # Convert to base64
+            buffered = BytesIO()
+            img.save(buffered, format="JPEG")
+            img_base64 = base64.b64encode(buffered.getvalue()).decode('utf-8')
+
+            return img_base64
+        
+    def quit(self):
+        self.driver.quit()

crawl4ai/database.py

@@ -0,0 +1,135 @@
+import os
+from pathlib import Path
+import sqlite3
+from typing import Optional, Tuple
+
+DB_PATH = os.path.join(os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai")
+os.makedirs(DB_PATH, exist_ok=True)
+DB_PATH = os.path.join(DB_PATH, "crawl4ai.db")
+
+def init_db():
+    global DB_PATH
+    conn = sqlite3.connect(DB_PATH)
+    cursor = conn.cursor()
+    cursor.execute('''
+        CREATE TABLE IF NOT EXISTS crawled_data (
+            url TEXT PRIMARY KEY,
+            html TEXT,
+            cleaned_html TEXT,
+            markdown TEXT,
+            extracted_content TEXT,
+            success BOOLEAN,
+            media TEXT DEFAULT "{}",
+            links TEXT DEFAULT "{}",
+            metadata TEXT DEFAULT "{}",
+            screenshot TEXT DEFAULT ""
+        )
+    ''')
+    conn.commit()
+    conn.close()
+
+def alter_db_add_screenshot(new_column: str = "media"):
+    check_db_path()
+    try:
+        conn = sqlite3.connect(DB_PATH)
+        cursor = conn.cursor()
+        cursor.execute(f'ALTER TABLE crawled_data ADD COLUMN {new_column} TEXT DEFAULT ""')
+        conn.commit()
+        conn.close()
+    except Exception as e:
+        print(f"Error altering database to add screenshot column: {e}")
+
+def check_db_path():
+    if not DB_PATH:
+        raise ValueError("Database path is not set or is empty.")
+
+def get_cached_url(url: str) -> Optional[Tuple[str, str, str, str, str, str, str, bool, str]]:
+    check_db_path()
+    try:
+        conn = sqlite3.connect(DB_PATH)
+        cursor = conn.cursor()
+        cursor.execute('SELECT url, html, cleaned_html, markdown, extracted_content, success, media, links, metadata, screenshot FROM crawled_data WHERE url = ?', (url,))
+        result = cursor.fetchone()
+        conn.close()
+        return result
+    except Exception as e:
+        print(f"Error retrieving cached URL: {e}")
+        return None
+
+def cache_url(url: str, html: str, cleaned_html: str, markdown: str, extracted_content: str, success: bool, media : str = "{}", links : str = "{}", metadata : str = "{}", screenshot: str = ""):
+    check_db_path()
+    try:
+        conn = sqlite3.connect(DB_PATH)
+        cursor = conn.cursor()
+        cursor.execute('''
+            INSERT INTO crawled_data (url, html, cleaned_html, markdown, extracted_content, success, media, links, metadata, screenshot)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            ON CONFLICT(url) DO UPDATE SET
+                html = excluded.html,
+                cleaned_html = excluded.cleaned_html,
+                markdown = excluded.markdown,
+                extracted_content = excluded.extracted_content,
+                success = excluded.success,
+                media = excluded.media,      
+                links = excluded.links,    
+                metadata = excluded.metadata,      
+                screenshot = excluded.screenshot
+        ''', (url, html, cleaned_html, markdown, extracted_content, success, media, links, metadata, screenshot))
+        conn.commit()
+        conn.close()
+    except Exception as e:
+        print(f"Error caching URL: {e}")
+
+def get_total_count() -> int:
+    check_db_path()
+    try:
+        conn = sqlite3.connect(DB_PATH)
+        cursor = conn.cursor()
+        cursor.execute('SELECT COUNT(*) FROM crawled_data')
+        result = cursor.fetchone()
+        conn.close()
+        return result[0]
+    except Exception as e:
+        print(f"Error getting total count: {e}")
+        return 0
+
+def clear_db():
+    check_db_path()
+    try:
+        conn = sqlite3.connect(DB_PATH)
+        cursor = conn.cursor()
+        cursor.execute('DELETE FROM crawled_data')
+        conn.commit()
+        conn.close()
+    except Exception as e:
+        print(f"Error clearing database: {e}")
+        
+def flush_db():
+    check_db_path()
+    try:
+        conn = sqlite3.connect(DB_PATH)
+        cursor = conn.cursor()
+        cursor.execute('DROP TABLE crawled_data')
+        conn.commit()
+        conn.close()
+    except Exception as e:
+        print(f"Error flushing database: {e}")
+
+def update_existing_records(new_column: str = "media", default_value: str = "{}"):
+    check_db_path()
+    try:
+        conn = sqlite3.connect(DB_PATH)
+        cursor = conn.cursor()
+        cursor.execute(f'UPDATE crawled_data SET {new_column} = "{default_value}" WHERE screenshot IS NULL')
+        conn.commit()
+        conn.close()
+    except Exception as e:
+        print(f"Error updating existing records: {e}")
+
+if __name__ == "__main__":
+    # Delete the existing database file
+    if os.path.exists(DB_PATH):
+        os.remove(DB_PATH)
+    init_db()  
+    # alter_db_add_screenshot("COL_NAME")
+    

crawl4ai/docs_manager.py

@@ -0,0 +1,67 @@
+import requests
+import shutil
+from pathlib import Path
+from crawl4ai.async_logger import AsyncLogger
+from crawl4ai.llmtxt import AsyncLLMTextManager
+
+class DocsManager:
+    def __init__(self, logger=None):
+        self.docs_dir = Path.home() / ".crawl4ai" / "docs"
+        self.local_docs = Path(__file__).parent.parent / "docs" / "llm.txt"
+        self.docs_dir.mkdir(parents=True, exist_ok=True)
+        self.logger = logger or AsyncLogger(verbose=True)
+        self.llm_text = AsyncLLMTextManager(self.docs_dir, self.logger)
+
+    async def ensure_docs_exist(self):
+        """Fetch docs if not present"""
+        if not any(self.docs_dir.iterdir()):
+            await self.fetch_docs()
+
+    async def fetch_docs(self) -> bool:
+        """Copy from local docs or download from GitHub"""
+        try:
+            # Try local first
+            if self.local_docs.exists() and (any(self.local_docs.glob("*.md")) or any(self.local_docs.glob("*.tokens"))):
+                # Empty the local docs directory
+                for file_path in self.docs_dir.glob("*.md"):
+                    file_path.unlink()
+                # for file_path in self.docs_dir.glob("*.tokens"): 
+                #     file_path.unlink()
+                for file_path in self.local_docs.glob("*.md"):
+                    shutil.copy2(file_path, self.docs_dir / file_path.name)
+                # for file_path in self.local_docs.glob("*.tokens"):
+                #     shutil.copy2(file_path, self.docs_dir / file_path.name)
+                return True
+
+            # Fallback to GitHub
+            response = requests.get(
+                "https://api.github.com/repos/unclecode/crawl4ai/contents/docs/llm.txt",
+                headers={'Accept': 'application/vnd.github.v3+json'}
+            )
+            response.raise_for_status()
+            
+            for item in response.json():
+                if item['type'] == 'file' and item['name'].endswith('.md'):
+                    content = requests.get(item['download_url']).text
+                    with open(self.docs_dir / item['name'], 'w', encoding='utf-8') as f:
+                        f.write(content)
+            return True
+
+        except Exception as e:
+            self.logger.error(f"Failed to fetch docs: {str(e)}")
+            raise
+
+    def list(self) -> list[str]:
+        """List available topics"""
+        names = [file_path.stem for file_path in self.docs_dir.glob("*.md")]
+        # Remove [0-9]+_ prefix
+        names = [name.split("_", 1)[1] if name[0].isdigit() else name for name in names]
+        # Exclude those end with .xs.md and .q.md
+        names = [name for name in names if not name.endswith(".xs") and not name.endswith(".q")]
+        return names
+    
+    def generate(self, sections, mode="extended"):
+        return self.llm_text.generate(sections, mode)
+    
+    def search(self, query: str, top_k: int = 5):
+        return self.llm_text.search(query, top_k)

crawl4ai/extraction_strategy.bak.py

@@ -0,0 +1,1440 @@
+from abc import ABC, abstractmethod
+from typing import Any, List, Dict, Optional, Union
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import json, time
+# from optimum.intel import IPEXModel
+from .prompts import *
+from .config import *
+from .utils import *
+from .models import *
+from functools import partial
+from .model_loader import *
+import math
+import numpy as np
+import re
+from bs4 import BeautifulSoup
+from lxml import html, etree
+from dataclasses import dataclass
+
+class ExtractionStrategy(ABC):
+    """
+    Abstract base class for all extraction strategies.
+    """
+    
+    def __init__(self, input_format: str = "markdown", **kwargs):
+        """
+        Initialize the extraction strategy.
+
+        Args:
+            input_format: Content format to use for extraction.
+                         Options: "markdown" (default), "html", "fit_markdown"
+            **kwargs: Additional keyword arguments
+        """
+        self.input_format = input_format
+        self.DEL = "<|DEL|>"
+        self.name = self.__class__.__name__
+        self.verbose = kwargs.get("verbose", False)
+
+    @abstractmethod
+    def extract(self, url: str, html: str, *q, **kwargs) -> List[Dict[str, Any]]:
+        """
+        Extract meaningful blocks or chunks from the given HTML.
+
+        :param url: The URL of the webpage.
+        :param html: The HTML content of the webpage.
+        :return: A list of extracted blocks or chunks.
+        """
+        pass
+    
+    def run(self, url: str, sections: List[str], *q, **kwargs) -> List[Dict[str, Any]]:
+        """
+        Process sections of text in parallel by default.
+
+        :param url: The URL of the webpage.
+        :param sections: List of sections (strings) to process.
+        :return: A list of processed JSON blocks.
+        """
+        extracted_content = []
+        with ThreadPoolExecutor() as executor:
+            futures = [executor.submit(self.extract, url, section, **kwargs) for section in sections]
+            for future in as_completed(futures):
+                extracted_content.extend(future.result())
+        return extracted_content    
+    
+class NoExtractionStrategy(ExtractionStrategy):
+    """
+    A strategy that does not extract any meaningful content from the HTML. It simply returns the entire HTML as a single block.
+    """
+    def extract(self, url: str, html: str, *q, **kwargs) -> List[Dict[str, Any]]:
+        """
+        Extract meaningful blocks or chunks from the given HTML.
+        """
+        return [{"index": 0, "content": html}]
+    
+    def run(self, url: str, sections: List[str], *q, **kwargs) -> List[Dict[str, Any]]:
+        return [{"index": i, "tags": [], "content": section} for i, section in enumerate(sections)]
+
+#######################################################
+# Strategies using LLM-based extraction for text data #
+#######################################################
+class LLMExtractionStrategy(ExtractionStrategy):
+    """
+    A strategy that uses an LLM to extract meaningful content from the HTML.
+    
+    Attributes:
+        provider: The provider to use for extraction. It follows the format <provider_name>/<model_name>, e.g., "ollama/llama3.3".
+        api_token: The API token for the provider.
+        instruction: The instruction to use for the LLM model.  
+        schema: Pydantic model schema for structured data.
+        extraction_type: "block" or "schema".
+        chunk_token_threshold: Maximum tokens per chunk.
+        overlap_rate: Overlap between chunks.
+        word_token_rate: Word to token conversion rate.
+        apply_chunking: Whether to apply chunking.
+        base_url: The base URL for the API request.
+        api_base: The base URL for the API request.
+        extra_args: Additional arguments for the API request, such as temprature, max_tokens, etc.
+        verbose: Whether to print verbose output.
+        usages: List of individual token usages.
+        total_usage: Accumulated token usage.
+    """
+
+    def __init__(self, 
+                 provider: str = DEFAULT_PROVIDER, api_token: Optional[str] = None, 
+                 instruction:str = None, schema:Dict = None, extraction_type = "block", **kwargs):
+        """
+        Initialize the strategy with clustering parameters.
+        
+        Args:
+            provider: The provider to use for extraction. It follows the format <provider_name>/<model_name>, e.g., "ollama/llama3.3".
+            api_token: The API token for the provider.
+            instruction: The instruction to use for the LLM model.  
+            schema: Pydantic model schema for structured data.
+            extraction_type: "block" or "schema".
+            chunk_token_threshold: Maximum tokens per chunk.
+            overlap_rate: Overlap between chunks.
+            word_token_rate: Word to token conversion rate.
+            apply_chunking: Whether to apply chunking.
+            base_url: The base URL for the API request.
+            api_base: The base URL for the API request.
+            extra_args: Additional arguments for the API request, such as temprature, max_tokens, etc.
+            verbose: Whether to print verbose output.
+            usages: List of individual token usages.
+            total_usage: Accumulated token usage.   
+
+        """
+        super().__init__(**kwargs)
+        self.provider = provider
+        self.api_token = api_token or PROVIDER_MODELS.get(provider, "no-token") or os.getenv("OPENAI_API_KEY")
+        self.instruction = instruction
+        self.extract_type = extraction_type
+        self.schema = schema
+        if schema:
+            self.extract_type = "schema"
+        
+        self.chunk_token_threshold = kwargs.get("chunk_token_threshold", CHUNK_TOKEN_THRESHOLD)
+        self.overlap_rate = kwargs.get("overlap_rate", OVERLAP_RATE)
+        self.word_token_rate = kwargs.get("word_token_rate", WORD_TOKEN_RATE)
+        self.apply_chunking = kwargs.get("apply_chunking", True)
+        self.base_url = kwargs.get("base_url", None)
+        self.api_base = kwargs.get("api_base", kwargs.get("base_url", None))
+        self.extra_args = kwargs.get("extra_args", {})
+        if not self.apply_chunking:
+            self.chunk_token_threshold = 1e9
+        
+        self.verbose = kwargs.get("verbose", False)
+        self.usages = []  # Store individual usages
+        self.total_usage = TokenUsage()  # Accumulated usage        
+        
+        if not self.api_token:
+            raise ValueError("API token must be provided for LLMExtractionStrategy. Update the config.py or set OPENAI_API_KEY environment variable.")
+        
+            
+    def extract(self, url: str, ix:int, html: str) -> List[Dict[str, Any]]:
+        """
+        Extract meaningful blocks or chunks from the given HTML using an LLM.
+        
+        How it works:
+        1. Construct a prompt with variables.
+        2. Make a request to the LLM using the prompt.
+        3. Parse the response and extract blocks or chunks.
+        
+        Args:
+            url: The URL of the webpage.
+            ix: Index of the block.
+            html: The HTML content of the webpage.
+            
+        Returns:
+            A list of extracted blocks or chunks.
+        """
+        if self.verbose:
+            # print("[LOG] Extracting blocks from URL:", url)
+            print(f"[LOG] Call LLM for {url} - block index: {ix}")
+
+        variable_values = {
+            "URL": url,
+            "HTML": escape_json_string(sanitize_html(html)),
+        }
+        
+        prompt_with_variables = PROMPT_EXTRACT_BLOCKS
+        if self.instruction:
+            variable_values["REQUEST"] = self.instruction
+            prompt_with_variables = PROMPT_EXTRACT_BLOCKS_WITH_INSTRUCTION
+            
+        if self.extract_type == "schema" and self.schema:
+            variable_values["SCHEMA"] = json.dumps(self.schema, indent=2)
+            prompt_with_variables = PROMPT_EXTRACT_SCHEMA_WITH_INSTRUCTION
+
+        for variable in variable_values:
+            prompt_with_variables = prompt_with_variables.replace(
+                "{" + variable + "}", variable_values[variable]
+            )
+        
+        response = perform_completion_with_backoff(
+            self.provider, 
+            prompt_with_variables, 
+            self.api_token, 
+            base_url=self.api_base or self.base_url,
+            extra_args = self.extra_args
+            ) # , json_response=self.extract_type == "schema")
+        # Track usage
+        usage = TokenUsage(
+            completion_tokens=response.usage.completion_tokens,
+            prompt_tokens=response.usage.prompt_tokens,
+            total_tokens=response.usage.total_tokens,
+            completion_tokens_details=response.usage.completion_tokens_details.__dict__ if response.usage.completion_tokens_details else {},
+            prompt_tokens_details=response.usage.prompt_tokens_details.__dict__ if response.usage.prompt_tokens_details else {}
+        )
+        self.usages.append(usage)
+        
+        # Update totals
+        self.total_usage.completion_tokens += usage.completion_tokens
+        self.total_usage.prompt_tokens += usage.prompt_tokens 
+        self.total_usage.total_tokens += usage.total_tokens
+        
+        try:
+            blocks = extract_xml_data(["blocks"], response.choices[0].message.content)['blocks']
+            blocks = json.loads(blocks)
+            for block in blocks:
+                block['error'] = False
+        except Exception as e:
+            parsed, unparsed = split_and_parse_json_objects(response.choices[0].message.content)
+            blocks = parsed
+            if unparsed:
+                blocks.append({
+                    "index": 0,
+                    "error": True,
+                    "tags": ["error"],
+                    "content": unparsed
+                })
+        
+        if self.verbose:
+            print("[LOG] Extracted", len(blocks), "blocks from URL:", url, "block index:", ix)
+        return blocks
+    
+    def _merge(self, documents, chunk_token_threshold, overlap):
+        """
+        Merge documents into sections based on chunk_token_threshold and overlap.
+        """
+        chunks = []
+        sections = []
+        total_tokens = 0
+
+        # Calculate the total tokens across all documents
+        for document in documents:
+            total_tokens += len(document.split(' ')) * self.word_token_rate
+
+        # Calculate the number of sections needed
+        num_sections = math.floor(total_tokens / chunk_token_threshold)
+        if num_sections < 1:
+            num_sections = 1  # Ensure there is at least one section
+        adjusted_chunk_threshold = total_tokens / num_sections
+
+        total_token_so_far = 0
+        current_chunk = []
+
+        for document in documents:
+            tokens = document.split(' ')
+            token_count = len(tokens) * self.word_token_rate
+            
+            if total_token_so_far + token_count <= adjusted_chunk_threshold:
+                current_chunk.extend(tokens)
+                total_token_so_far += token_count
+            else:
+                # Ensure to handle the last section properly
+                if len(sections) == num_sections - 1:
+                    current_chunk.extend(tokens)
+                    continue
+                
+                # Add overlap if specified
+                if overlap > 0 and current_chunk:
+                    overlap_tokens = current_chunk[-overlap:]
+                    current_chunk.extend(overlap_tokens)
+                
+                sections.append(' '.join(current_chunk))
+                current_chunk = tokens
+                total_token_so_far = token_count
+
+        # Add the last chunk
+        if current_chunk:
+            sections.append(' '.join(current_chunk))
+
+        return sections
+
+
+    def run(self, url: str, sections: List[str]) -> List[Dict[str, Any]]:
+        """
+        Process sections sequentially with a delay for rate limiting issues, specifically for LLMExtractionStrategy.
+        
+        Args:
+            url: The URL of the webpage.
+            sections: List of sections (strings) to process.
+            
+        Returns:
+            A list of extracted blocks or chunks.
+        """
+        
+        merged_sections = self._merge(
+            sections, self.chunk_token_threshold,
+            overlap= int(self.chunk_token_threshold * self.overlap_rate)
+        )
+        extracted_content = []
+        if self.provider.startswith("groq/"):
+            # Sequential processing with a delay
+            for ix, section in enumerate(merged_sections):
+                extract_func = partial(self.extract, url)
+                extracted_content.extend(extract_func(ix, sanitize_input_encode(section)))
+                time.sleep(0.5)  # 500 ms delay between each processing
+        else:
+            # Parallel processing using ThreadPoolExecutor
+            # extract_func = partial(self.extract, url)
+            # for ix, section in enumerate(merged_sections):
+            #     extracted_content.append(extract_func(ix, section))            
+            
+            with ThreadPoolExecutor(max_workers=4) as executor:
+                extract_func = partial(self.extract, url)
+                futures = [executor.submit(extract_func, ix, sanitize_input_encode(section)) for ix, section in enumerate(merged_sections)]
+                
+                for future in as_completed(futures):
+                    try:
+                        extracted_content.extend(future.result())
+                    except Exception as e:
+                        if self.verbose:
+                            print(f"Error in thread execution: {e}")
+                        # Add error information to extracted_content
+                        extracted_content.append({
+                            "index": 0,
+                            "error": True,
+                            "tags": ["error"],
+                            "content": str(e)
+                        })
+
+        
+        return extracted_content        
+    
+    
+    def show_usage(self) -> None:
+        """Print a detailed token usage report showing total and per-request usage."""
+        print("\n=== Token Usage Summary ===")
+        print(f"{'Type':<15} {'Count':>12}")
+        print("-" * 30)
+        print(f"{'Completion':<15} {self.total_usage.completion_tokens:>12,}")
+        print(f"{'Prompt':<15} {self.total_usage.prompt_tokens:>12,}")
+        print(f"{'Total':<15} {self.total_usage.total_tokens:>12,}")
+
+        print("\n=== Usage History ===")
+        print(f"{'Request #':<10} {'Completion':>12} {'Prompt':>12} {'Total':>12}")
+        print("-" * 48)
+        for i, usage in enumerate(self.usages, 1):
+            print(f"{i:<10} {usage.completion_tokens:>12,} {usage.prompt_tokens:>12,} {usage.total_tokens:>12,}")
+  
+#######################################################
+# Strategies using clustering for text data extraction #
+#######################################################
+
+class CosineStrategy(ExtractionStrategy):
+    """
+    Extract meaningful blocks or chunks from the given HTML using cosine similarity.
+    
+    How it works:
+    1. Pre-filter documents using embeddings and semantic_filter.
+    2. Perform clustering using cosine similarity.
+    3. Organize texts by their cluster labels, retaining order.
+    4. Filter clusters by word count.
+    5. Extract meaningful blocks or chunks from the filtered clusters.
+    
+    Attributes:
+        semantic_filter (str): A keyword filter for document filtering.
+        word_count_threshold (int): Minimum number of words per cluster.
+        max_dist (float): The maximum cophenetic distance on the dendrogram to form clusters.
+        linkage_method (str): The linkage method for hierarchical clustering.
+        top_k (int): Number of top categories to extract.
+        model_name (str): The name of the sentence-transformers model.
+        sim_threshold (float): The similarity threshold for clustering.
+    """ 
+    def __init__(self, semantic_filter = None, word_count_threshold=10, max_dist=0.2, linkage_method='ward', top_k=3, model_name = 'sentence-transformers/all-MiniLM-L6-v2', sim_threshold = 0.3, **kwargs):
+        """
+        Initialize the strategy with clustering parameters.
+
+        Args:
+            semantic_filter (str): A keyword filter for document filtering.
+            word_count_threshold (int): Minimum number of words per cluster.
+            max_dist (float): The maximum cophenetic distance on the dendrogram to form clusters.
+            linkage_method (str): The linkage method for hierarchical clustering.
+            top_k (int): Number of top categories to extract.
+        """
+        super().__init__(**kwargs)
+        
+        import numpy as np
+        
+        self.semantic_filter = semantic_filter
+        self.word_count_threshold = word_count_threshold
+        self.max_dist = max_dist
+        self.linkage_method = linkage_method
+        self.top_k = top_k
+        self.sim_threshold = sim_threshold
+        self.timer = time.time()
+        self.verbose = kwargs.get("verbose", False)
+        
+        self.buffer_embeddings = np.array([])
+        self.get_embedding_method = "direct"
+        
+        self.device = get_device()
+        # import torch
+        # self.device = torch.device('cpu')
+        
+        self.default_batch_size = calculate_batch_size(self.device)
+
+        if self.verbose:
+            print(f"[LOG] Loading Extraction Model for {self.device.type} device.")
+
+        # if False and self.device.type == "cpu":
+        #     self.model = load_onnx_all_MiniLM_l6_v2()
+        #     self.tokenizer = self.model.tokenizer
+        #     self.get_embedding_method = "direct"
+        # else:
+
+        self.tokenizer, self.model = load_HF_embedding_model(model_name)
+        self.model.to(self.device)
+        self.model.eval()  
+        
+        self.get_embedding_method = "batch"
+        
+        self.buffer_embeddings = np.array([])
+
+        # if model_name == "bert-base-uncased":
+        #     self.tokenizer, self.model = load_bert_base_uncased()
+        #     self.model.eval()  # Ensure the model is in evaluation mode
+        #     self.get_embedding_method = "batch"
+        # elif model_name == "BAAI/bge-small-en-v1.5":
+        #     self.tokenizer, self.model = load_bge_small_en_v1_5()
+        #     self.model.eval()  # Ensure the model is in evaluation mode
+        #     self.get_embedding_method = "batch"
+        # elif model_name == "sentence-transformers/all-MiniLM-L6-v2":
+        #     self.model = load_onnx_all_MiniLM_l6_v2()
+        #     self.tokenizer = self.model.tokenizer
+        #     self.get_embedding_method = "direct"
+       
+        
+        if self.verbose:
+            print(f"[LOG] Loading Multilabel Classifier for {self.device.type} device.")
+            
+        self.nlp, _ = load_text_multilabel_classifier()
+        # self.default_batch_size = 16 if self.device.type == 'cpu' else 64
+        
+        if self.verbose:
+            print(f"[LOG] Model loaded {model_name}, models/reuters, took " + str(time.time() - self.timer) + " seconds")
+
+    def filter_documents_embeddings(self, documents: List[str], semantic_filter: str, at_least_k: int = 20) -> List[str]:
+        """
+        Filter and sort documents based on the cosine similarity of their embeddings with the semantic_filter embedding.
+
+        Args:
+            documents (List[str]): A list of document texts.
+            semantic_filter (str): A keyword filter for document filtering.
+            at_least_k (int): The minimum number of documents to return.
+
+        Returns:
+            List[str]: A list of filtered and sorted document texts.
+        """
+        
+        if not semantic_filter:
+            return documents
+        
+        if len(documents) < at_least_k:
+            at_least_k = len(documents) // 2
+        
+        from sklearn.metrics.pairwise import cosine_similarity
+        
+        # Compute embedding for the keyword filter
+        query_embedding = self.get_embeddings([semantic_filter])[0]
+        
+        # Compute embeddings for the documents
+        document_embeddings = self.get_embeddings(documents)
+        
+        # Calculate cosine similarity between the query embedding and document embeddings
+        similarities = cosine_similarity([query_embedding], document_embeddings).flatten()
+        
+        # Filter documents based on the similarity threshold
+        filtered_docs = [(doc, sim) for doc, sim in zip(documents, similarities) if sim >= self.sim_threshold]
+        
+        # If the number of filtered documents is less than at_least_k, sort remaining documents by similarity
+        if len(filtered_docs) < at_least_k:
+            remaining_docs = [(doc, sim) for doc, sim in zip(documents, similarities) if sim < self.sim_threshold]
+            remaining_docs.sort(key=lambda x: x[1], reverse=True)
+            filtered_docs.extend(remaining_docs[:at_least_k - len(filtered_docs)])
+        
+        # Extract the document texts from the tuples
+        filtered_docs = [doc for doc, _ in filtered_docs]
+        
+        return filtered_docs[:at_least_k]
+    
+    def get_embeddings(self, sentences: List[str], batch_size=None, bypass_buffer=False):
+        """
+        Get BERT embeddings for a list of sentences.
+
+        Args:
+            sentences (List[str]): A list of text chunks (sentences).
+
+        Returns:
+            NumPy array of embeddings.
+        """
+        # if self.buffer_embeddings.any() and not bypass_buffer:
+        #     return self.buffer_embeddings
+        
+        if self.device.type in [ "cpu", "gpu", "cuda", "mps"]:
+            import torch 
+            # Tokenize sentences and convert to tensor
+            if batch_size is None:
+                batch_size = self.default_batch_size
+                        
+            all_embeddings = []
+            for i in range(0, len(sentences), batch_size):
+                batch_sentences = sentences[i:i + batch_size]
+                encoded_input = self.tokenizer(batch_sentences, padding=True, truncation=True, return_tensors='pt')
+                encoded_input = {key: tensor.to(self.device) for key, tensor in encoded_input.items()}
+                
+                # Ensure no gradients are calculated
+                with torch.no_grad():
+                    model_output = self.model(**encoded_input)
+                
+                # Get embeddings from the last hidden state (mean pooling)
+                embeddings = model_output.last_hidden_state.mean(dim=1).cpu().numpy()
+                all_embeddings.append(embeddings)
+            
+            self.buffer_embeddings = np.vstack(all_embeddings)
+        elif self.device.type == "cpu":      
+            # self.buffer_embeddings = self.model(sentences)
+            if batch_size is None:
+                batch_size = self.default_batch_size
+                
+            all_embeddings = []
+            for i in range(0, len(sentences), batch_size):
+                batch_sentences = sentences[i:i + batch_size]
+                embeddings = self.model(batch_sentences)
+                all_embeddings.append(embeddings)
+                
+            self.buffer_embeddings = np.vstack(all_embeddings)
+        return self.buffer_embeddings
+
+    def hierarchical_clustering(self, sentences: List[str], embeddings = None):
+        """
+        Perform hierarchical clustering on sentences and return cluster labels.
+
+        Args:
+            sentences (List[str]): A list of text chunks (sentences).
+
+        Returns:
+            NumPy array of cluster labels.
+        """
+        # Get embeddings
+        from scipy.cluster.hierarchy import linkage, fcluster
+        from scipy.spatial.distance import pdist
+        self.timer = time.time()
+        embeddings = self.get_embeddings(sentences, bypass_buffer=True)
+        # print(f"[LOG] 🚀 Embeddings computed in {time.time() - self.timer:.2f} seconds")
+        # Compute pairwise cosine distances
+        distance_matrix = pdist(embeddings, 'cosine')
+        # Perform agglomerative clustering respecting order
+        linked = linkage(distance_matrix, method=self.linkage_method)
+        # Form flat clusters
+        labels = fcluster(linked, self.max_dist, criterion='distance')
+        return labels
+
+    def filter_clusters_by_word_count(self, clusters: Dict[int, List[str]]) -> Dict[int, List[str]]:
+        """
+        Filter clusters to remove those with a word count below the threshold.
+
+        Args:
+            clusters (Dict[int, List[str]]): Dictionary of clusters.
+
+        Returns:
+            Dict[int, List[str]]: Filtered dictionary of clusters.
+        """
+        filtered_clusters = {}
+        for cluster_id, texts in clusters.items():
+            # Concatenate texts for analysis
+            full_text = " ".join(texts)
+            # Count words
+            word_count = len(full_text.split())
+            
+            # Keep clusters with word count above the threshold
+            if word_count >= self.word_count_threshold:
+                filtered_clusters[cluster_id] = texts
+
+        return filtered_clusters
+
+    def extract(self, url: str, html: str, *q, **kwargs) -> List[Dict[str, Any]]:
+        """
+        Extract clusters from HTML content using hierarchical clustering.
+
+        Args:
+            url (str): The URL of the webpage.
+            html (str): The HTML content of the webpage.
+
+        Returns:
+            List[Dict[str, Any]]: A list of processed JSON blocks.
+        """
+        # Assume `html` is a list of text chunks for this strategy
+        t = time.time()
+        text_chunks = html.split(self.DEL)  # Split by lines or paragraphs as needed
+        
+        # Pre-filter documents using embeddings and semantic_filter
+        text_chunks = self.filter_documents_embeddings(text_chunks, self.semantic_filter)
+
+        if not text_chunks:
+            return []
+
+        # Perform clustering
+        labels = self.hierarchical_clustering(text_chunks)
+        # print(f"[LOG] 🚀 Clustering done in {time.time() - t:.2f} seconds")
+
+        # Organize texts by their cluster labels, retaining order
+        t = time.time()
+        clusters = {}
+        for index, label in enumerate(labels):
+            clusters.setdefault(label, []).append(text_chunks[index])
+
+        # Filter clusters by word count
+        filtered_clusters = self.filter_clusters_by_word_count(clusters)
+
+        # Convert filtered clusters to a sorted list of dictionaries
+        cluster_list = [{"index": int(idx), "tags" : [], "content": " ".join(filtered_clusters[idx])} for idx in sorted(filtered_clusters)]
+        
+        if self.verbose:
+            print(f"[LOG] 🚀 Assign tags using {self.device}")
+        
+        if self.device.type in ["gpu", "cuda", "mps", "cpu"]:
+            labels = self.nlp([cluster['content'] for cluster in cluster_list])
+            
+            for cluster, label in zip(cluster_list, labels):
+                cluster['tags'] = label
+        # elif self.device.type == "cpu":
+        #     # Process the text with the loaded model
+        #     texts = [cluster['content'] for cluster in cluster_list]
+        #     # Batch process texts
+        #     docs = self.nlp.pipe(texts, disable=["tagger", "parser", "ner", "lemmatizer"])
+
+        #     for doc, cluster in zip(docs, cluster_list):
+        #         tok_k = self.top_k
+        #         top_categories = sorted(doc.cats.items(), key=lambda x: x[1], reverse=True)[:tok_k]
+        #         cluster['tags'] = [cat for cat, _ in top_categories]
+                            
+            # for cluster in  cluster_list:
+            #     doc = self.nlp(cluster['content'])
+            #     tok_k = self.top_k
+            #     top_categories = sorted(doc.cats.items(), key=lambda x: x[1], reverse=True)[:tok_k]
+            #     cluster['tags'] = [cat for cat, _ in top_categories]
+        
+        if self.verbose:
+            print(f"[LOG] 🚀 Categorization done in {time.time() - t:.2f} seconds")
+        
+        return cluster_list
+
+    def run(self, url: str, sections: List[str], *q, **kwargs) -> List[Dict[str, Any]]:
+        """
+        Process sections using hierarchical clustering.
+
+        Args:
+            url (str): The URL of the webpage.
+            sections (List[str]): List of sections (strings) to process.
+
+        Returns:
+        """
+        # This strategy processes all sections together
+        
+        return self.extract(url, self.DEL.join(sections), **kwargs)
+    
+#######################################################
+# New extraction strategies for JSON-based extraction #
+####################################################### 
+
+class JsonElementExtractionStrategy(ExtractionStrategy):
+    """
+    Abstract base class for extracting structured JSON from HTML content.
+
+    How it works:
+    1. Parses HTML content using the `_parse_html` method.
+    2. Uses a schema to define base selectors, fields, and transformations.
+    3. Extracts data hierarchically, supporting nested fields and lists.
+    4. Handles computed fields with expressions or functions.
+
+    Attributes:
+        DEL (str): Delimiter used to combine HTML sections. Defaults to '\n'.
+        schema (Dict[str, Any]): The schema defining the extraction rules.
+        verbose (bool): Enables verbose logging for debugging purposes.
+
+    Methods:
+        extract(url, html_content, *q, **kwargs): Extracts structured data from HTML content.
+        _extract_item(element, fields): Extracts fields from a single element.
+        _extract_single_field(element, field): Extracts a single field based on its type.
+        _apply_transform(value, transform): Applies a transformation to a value.
+        _compute_field(item, field): Computes a field value using an expression or function.
+        run(url, sections, *q, **kwargs): Combines HTML sections and runs the extraction strategy.
+
+    Abstract Methods:
+        _parse_html(html_content): Parses raw HTML into a structured format (e.g., BeautifulSoup or lxml).
+        _get_base_elements(parsed_html, selector): Retrieves base elements using a selector.
+        _get_elements(element, selector): Retrieves child elements using a selector.
+        _get_element_text(element): Extracts text content from an element.
+        _get_element_html(element): Extracts raw HTML from an element.
+        _get_element_attribute(element, attribute): Extracts an attribute's value from an element.
+    """
+
+    
+    DEL = '\n'
+
+    def __init__(self, schema: Dict[str, Any], **kwargs):
+        """
+        Initialize the JSON element extraction strategy with a schema.
+
+        Args:
+            schema (Dict[str, Any]): The schema defining the extraction rules.
+        """
+        super().__init__(**kwargs)
+        self.schema = schema
+        self.verbose = kwargs.get('verbose', False)
+
+    def extract(self, url: str, html_content: str, *q, **kwargs) -> List[Dict[str, Any]]:
+        """
+        Extract structured data from HTML content.
+
+        How it works:
+        1. Parses the HTML content using the `_parse_html` method.
+        2. Identifies base elements using the schema's base selector.
+        3. Extracts fields from each base element using `_extract_item`.
+
+        Args:
+            url (str): The URL of the page being processed.
+            html_content (str): The raw HTML content to parse and extract.
+            *q: Additional positional arguments.
+            **kwargs: Additional keyword arguments for custom extraction.
+
+        Returns:
+            List[Dict[str, Any]]: A list of extracted items, each represented as a dictionary.
+        """
+        
+        parsed_html = self._parse_html(html_content)
+        base_elements = self._get_base_elements(parsed_html, self.schema['baseSelector'])
+        
+        results = []
+        for element in base_elements:
+            # Extract base element attributes
+            item = {}
+            if 'baseFields' in self.schema:
+                for field in self.schema['baseFields']:
+                    value = self._extract_single_field(element, field)
+                    if value is not None:
+                        item[field['name']] = value
+            
+            # Extract child fields
+            field_data = self._extract_item(element, self.schema['fields'])
+            item.update(field_data)
+            
+            if item:
+                results.append(item)
+        
+        return results
+
+    @abstractmethod
+    def _parse_html(self, html_content: str):
+        """Parse HTML content into appropriate format"""
+        pass
+
+    @abstractmethod
+    def _get_base_elements(self, parsed_html, selector: str):
+        """Get all base elements using the selector"""
+        pass
+
+    @abstractmethod
+    def _get_elements(self, element, selector: str):
+        """Get child elements using the selector"""
+        pass
+
+    def _extract_field(self, element, field):
+        try:
+            if field['type'] == 'nested':
+                nested_elements = self._get_elements(element, field['selector'])
+                nested_element = nested_elements[0] if nested_elements else None
+                return self._extract_item(nested_element, field['fields']) if nested_element else {}
+            
+            if field['type'] == 'list':
+                elements = self._get_elements(element, field['selector'])
+                return [self._extract_list_item(el, field['fields']) for el in elements]
+            
+            if field['type'] == 'nested_list':
+                elements = self._get_elements(element, field['selector'])
+                return [self._extract_item(el, field['fields']) for el in elements]
+            
+            return self._extract_single_field(element, field)
+        except Exception as e:
+            if self.verbose:
+                print(f"Error extracting field {field['name']}: {str(e)}")
+            return field.get('default')
+
+    def _extract_single_field(self, element, field):
+        """
+        Extract a single field based on its type.
+
+        How it works:
+        1. Selects the target element using the field's selector.
+        2. Extracts the field value based on its type (e.g., text, attribute, regex).
+        3. Applies transformations if defined in the schema.
+
+        Args:
+            element: The base element to extract the field from.
+            field (Dict[str, Any]): The field definition in the schema.
+
+        Returns:
+            Any: The extracted field value.
+        """
+        
+        if 'selector' in field:
+            selected = self._get_elements(element, field['selector'])
+            if not selected:
+                return field.get('default')
+            selected = selected[0]
+        else:
+            selected = element
+
+        value = None
+        if field['type'] == 'text':
+            value = self._get_element_text(selected)
+        elif field['type'] == 'attribute':
+            value = self._get_element_attribute(selected, field['attribute'])
+        elif field['type'] == 'html':
+            value = self._get_element_html(selected)
+        elif field['type'] == 'regex':
+            text = self._get_element_text(selected)
+            match = re.search(field['pattern'], text)
+            value = match.group(1) if match else None
+
+        if 'transform' in field:
+            value = self._apply_transform(value, field['transform'])
+
+        return value if value is not None else field.get('default')
+
+    def _extract_list_item(self, element, fields):
+        item = {}
+        for field in fields:
+            value = self._extract_single_field(element, field)
+            if value is not None:
+                item[field['name']] = value
+        return item
+
+    def _extract_item(self, element, fields):
+        """
+        Extracts fields from a given element.
+
+        How it works:
+        1. Iterates through the fields defined in the schema.
+        2. Handles computed, single, and nested field types.
+        3. Updates the item dictionary with extracted field values.
+
+        Args:
+            element: The base element to extract fields from.
+            fields (List[Dict[str, Any]]): The list of fields to extract.
+
+        Returns:
+            Dict[str, Any]: A dictionary representing the extracted item.
+        """
+        
+        item = {}
+        for field in fields:
+            if field['type'] == 'computed':
+                value = self._compute_field(item, field)
+            else:
+                value = self._extract_field(element, field)
+            if value is not None:
+                item[field['name']] = value
+        return item
+
+    def _apply_transform(self, value, transform):
+        """
+        Apply a transformation to a value.
+
+        How it works:
+        1. Checks the transformation type (e.g., `lowercase`, `strip`).
+        2. Applies the transformation to the value.
+        3. Returns the transformed value.
+
+        Args:
+            value (str): The value to transform.
+            transform (str): The type of transformation to apply.
+
+        Returns:
+            str: The transformed value.
+        """
+        
+        if transform == 'lowercase':
+            return value.lower()
+        elif transform == 'uppercase':
+            return value.upper()
+        elif transform == 'strip':
+            return value.strip()
+        return value
+
+    def _compute_field(self, item, field):
+        try:
+            if 'expression' in field:
+                return eval(field['expression'], {}, item)
+            elif 'function' in field:
+                return field['function'](item)
+        except Exception as e:
+            if self.verbose:
+                print(f"Error computing field {field['name']}: {str(e)}")
+            return field.get('default')
+
+    def run(self, url: str, sections: List[str], *q, **kwargs) -> List[Dict[str, Any]]:
+        """
+        Run the extraction strategy on a combined HTML content.
+
+        How it works:
+        1. Combines multiple HTML sections using the `DEL` delimiter.
+        2. Calls the `extract` method with the combined HTML.
+
+        Args:
+            url (str): The URL of the page being processed.
+            sections (List[str]): A list of HTML sections.
+            *q: Additional positional arguments.
+            **kwargs: Additional keyword arguments for custom extraction.
+
+        Returns:
+            List[Dict[str, Any]]: A list of extracted items.
+        """
+        
+        combined_html = self.DEL.join(sections)
+        return self.extract(url, combined_html, **kwargs)
+
+    @abstractmethod
+    def _get_element_text(self, element) -> str:
+        """Get text content from element"""
+        pass
+
+    @abstractmethod
+    def _get_element_html(self, element) -> str:
+        """Get HTML content from element"""
+        pass
+
+    @abstractmethod
+    def _get_element_attribute(self, element, attribute: str):
+        """Get attribute value from element"""
+        pass
+
+class JsonCssExtractionStrategy(JsonElementExtractionStrategy):
+    """
+    Concrete implementation of `JsonElementExtractionStrategy` using CSS selectors.
+
+    How it works:
+    1. Parses HTML content with BeautifulSoup.
+    2. Selects elements using CSS selectors defined in the schema.
+    3. Extracts field data and applies transformations as defined.
+
+    Attributes:
+        schema (Dict[str, Any]): The schema defining the extraction rules.
+        verbose (bool): Enables verbose logging for debugging purposes.
+
+    Methods:
+        _parse_html(html_content): Parses HTML content into a BeautifulSoup object.
+        _get_base_elements(parsed_html, selector): Selects base elements using a CSS selector.
+        _get_elements(element, selector): Selects child elements using a CSS selector.
+        _get_element_text(element): Extracts text content from a BeautifulSoup element.
+        _get_element_html(element): Extracts the raw HTML content of a BeautifulSoup element.
+        _get_element_attribute(element, attribute): Retrieves an attribute value from a BeautifulSoup element.
+    """
+    
+    def __init__(self, schema: Dict[str, Any], **kwargs):
+        kwargs['input_format'] = 'html'  # Force HTML input
+        super().__init__(schema, **kwargs)
+
+    def _parse_html(self, html_content: str):
+        return BeautifulSoup(html_content, 'html.parser')
+
+    def _get_base_elements(self, parsed_html, selector: str):
+        return parsed_html.select(selector)
+
+    def _get_elements(self, element, selector: str):
+        selected = element.select_one(selector)
+        return [selected] if selected else []
+
+    def _get_element_text(self, element) -> str:
+        return element.get_text(strip=True)
+
+    def _get_element_html(self, element) -> str:
+        return str(element)
+
+    def _get_element_attribute(self, element, attribute: str):
+        return element.get(attribute)
+
+class JsonXPathExtractionStrategy(JsonElementExtractionStrategy):
+    """
+    Concrete implementation of `JsonElementExtractionStrategy` using XPath selectors.
+
+    How it works:
+    1. Parses HTML content into an lxml tree.
+    2. Selects elements using XPath expressions.
+    3. Converts CSS selectors to XPath when needed.
+
+    Attributes:
+        schema (Dict[str, Any]): The schema defining the extraction rules.
+        verbose (bool): Enables verbose logging for debugging purposes.
+
+    Methods:
+        _parse_html(html_content): Parses HTML content into an lxml tree.
+        _get_base_elements(parsed_html, selector): Selects base elements using an XPath selector.
+        _css_to_xpath(css_selector): Converts a CSS selector to an XPath expression.
+        _get_elements(element, selector): Selects child elements using an XPath selector.
+        _get_element_text(element): Extracts text content from an lxml element.
+        _get_element_html(element): Extracts the raw HTML content of an lxml element.
+        _get_element_attribute(element, attribute): Retrieves an attribute value from an lxml element.
+    """
+    
+    def __init__(self, schema: Dict[str, Any], **kwargs):
+        kwargs['input_format'] = 'html'  # Force HTML input
+        super().__init__(schema, **kwargs)
+
+    def _parse_html(self, html_content: str):
+        return html.fromstring(html_content)
+
+    def _get_base_elements(self, parsed_html, selector: str):
+        return parsed_html.xpath(selector)
+
+    def _css_to_xpath(self, css_selector: str) -> str:
+        """Convert CSS selector to XPath if needed"""
+        if '/' in css_selector:  # Already an XPath
+            return css_selector
+        return self._basic_css_to_xpath(css_selector)
+
+    def _basic_css_to_xpath(self, css_selector: str) -> str:
+        """Basic CSS to XPath conversion for common cases"""
+        if ' > ' in css_selector:
+            parts = css_selector.split(' > ')
+            return '//' + '/'.join(parts)
+        if ' ' in css_selector:
+            parts = css_selector.split(' ')
+            return '//' + '//'.join(parts)
+        return '//' + css_selector
+
+    def _get_elements(self, element, selector: str):
+        xpath = self._css_to_xpath(selector)
+        if not xpath.startswith('.'):
+            xpath = '.' + xpath
+        return element.xpath(xpath)
+
+    def _get_element_text(self, element) -> str:
+        return ''.join(element.xpath('.//text()')).strip()
+
+    def _get_element_html(self, element) -> str:
+        return etree.tostring(element, encoding='unicode')
+
+    def _get_element_attribute(self, element, attribute: str):
+        return element.get(attribute)
+ 
+
+#######################################################
+# Strategies based on the extraction of specific types#
+#######################################################
+    
+class TopicExtractionStrategy(ExtractionStrategy):
+    def __init__(self, num_keywords: int = 3, **kwargs):
+        """
+        Initialize the topic extraction strategy with parameters for topic segmentation.
+
+        :param num_keywords: Number of keywords to represent each topic segment.
+        """
+        import nltk
+        super().__init__(**kwargs)
+        self.num_keywords = num_keywords
+        self.tokenizer = nltk.TextTilingTokenizer()
+
+    def extract_keywords(self, text: str) -> List[str]:
+        """
+        Extract keywords from a given text segment using simple frequency analysis.
+
+        :param text: The text segment from which to extract keywords.
+        :return: A list of keyword strings.
+        """
+        import nltk
+        # Tokenize the text and compute word frequency
+        words = nltk.word_tokenize(text)
+        freq_dist = nltk.FreqDist(words)
+        # Get the most common words as keywords
+        keywords = [word for (word, _) in freq_dist.most_common(self.num_keywords)]
+        return keywords
+
+    def extract(self, url: str, html: str, *q, **kwargs) -> List[Dict[str, Any]]:
+        """
+        Extract topics from HTML content using TextTiling for segmentation and keyword extraction.
+
+        :param url: The URL of the webpage.
+        :param html: The HTML content of the webpage.
+        :param provider: The provider to be used for extraction (not used here).
+        :param api_token: Optional API token for the provider (not used here).
+        :return: A list of dictionaries representing the topics.
+        """
+        # Use TextTiling to segment the text into topics
+        segmented_topics = html.split(self.DEL)  # Split by lines or paragraphs as needed
+
+        # Prepare the output as a list of dictionaries
+        topic_list = []
+        for i, segment in enumerate(segmented_topics):
+            # Extract keywords for each segment
+            keywords = self.extract_keywords(segment)
+            topic_list.append({
+                "index": i,
+                "content": segment,
+                "keywords": keywords
+            })
+
+        return topic_list
+
+    def run(self, url: str, sections: List[str], *q, **kwargs) -> List[Dict[str, Any]]:
+        """
+        Process sections using topic segmentation and keyword extraction.
+
+        :param url: The URL of the webpage.
+        :param sections: List of sections (strings) to process.
+        :param provider: The provider to be used for extraction (not used here).
+        :param api_token: Optional API token for the provider (not used here).
+        :return: A list of processed JSON blocks.
+        """
+        # Concatenate sections into a single text for coherent topic segmentation
+        
+        
+        return self.extract(url, self.DEL.join(sections), **kwargs)
+    
+class ContentSummarizationStrategy(ExtractionStrategy):
+    def __init__(self, model_name: str = "sshleifer/distilbart-cnn-12-6", **kwargs):
+        """
+        Initialize the content summarization strategy with a specific model.
+
+        :param model_name: The model to use for summarization.
+        """
+        super().__init__(**kwargs)
+        from transformers import pipeline
+        self.summarizer = pipeline("summarization", model=model_name)
+
+    def extract(self, url: str, text: str, provider: str = None, api_token: Optional[str] = None) -> List[Dict[str, Any]]:
+        """
+        Summarize a single section of text.
+
+        :param url: The URL of the webpage.
+        :param text: A section of text to summarize.
+        :param provider: The provider to be used for extraction (not used here).
+        :param api_token: Optional API token for the provider (not used here).
+        :return: A dictionary with the summary.
+        """
+        try:
+            summary = self.summarizer(text, max_length=130, min_length=30, do_sample=False)
+            return {"summary": summary[0]['summary_text']}
+        except Exception as e:
+            print(f"Error summarizing text: {e}")
+            return {"summary": text}  # Fallback to original text if summarization fails
+
+    def run(self, url: str, sections: List[str], provider: str = None, api_token: Optional[str] = None) -> List[Dict[str, Any]]:
+        """
+        Process each section in parallel to produce summaries.
+
+        :param url: The URL of the webpage.
+        :param sections: List of sections (strings) to summarize.
+        :param provider: The provider to be used for extraction (not used here).
+        :param api_token: Optional API token for the provider (not used here).
+        :return: A list of dictionaries with summaries for each section.
+        """
+        # Use a ThreadPoolExecutor to summarize in parallel
+        summaries = []
+        with ThreadPoolExecutor() as executor:
+            # Create a future for each section's summarization
+            future_to_section = {executor.submit(self.extract, url, section, provider, api_token): i for i, section in enumerate(sections)}
+            for future in as_completed(future_to_section):
+                section_index = future_to_section[future]
+                try:
+                    summary_result = future.result()
+                    summaries.append((section_index, summary_result))
+                except Exception as e:
+                    print(f"Error processing section {section_index}: {e}")
+                    summaries.append((section_index, {"summary": sections[section_index]}))  # Fallback to original text
+
+        # Sort summaries by the original section index to maintain order
+        summaries.sort(key=lambda x: x[0])
+        return [summary for _, summary in summaries]
+ 
+#######################################################
+# Deprecated strategies
+#######################################################
+ 
+class _JsonCssExtractionStrategy(ExtractionStrategy):
+    def __init__(self, schema: Dict[str, Any], **kwargs):
+        kwargs['input_format'] = 'html'  # Force HTML input
+        super().__init__(**kwargs)
+        self.schema = schema
+
+    def extract(self, url: str, html: str, *q, **kwargs) -> List[Dict[str, Any]]:
+        soup = BeautifulSoup(html, 'html.parser')
+        base_elements = soup.select(self.schema['baseSelector'])
+        
+        results = []
+        for element in base_elements:
+            # Extract base element attributes first
+            item = {}
+            if 'baseFields' in self.schema:
+                for field in self.schema['baseFields']:
+                    value = self._extract_single_field(element, field)
+                    if value is not None:
+                        item[field['name']] = value
+            
+            # Then extract child fields
+            field_data = self._extract_item(element, self.schema['fields'])
+            item.update(field_data)
+            
+            results.append(item)
+        
+        return results
+
+    def _extract_field(self, element, field):
+        try:
+            if field['type'] == 'nested':
+                nested_element = element.select_one(field['selector'])
+                return self._extract_item(nested_element, field['fields']) if nested_element else {}
+            
+            if field['type'] == 'list':
+                elements = element.select(field['selector'])
+                return [self._extract_list_item(el, field['fields']) for el in elements]
+            
+            if field['type'] == 'nested_list':
+                elements = element.select(field['selector'])
+                return [self._extract_item(el, field['fields']) for el in elements]
+            
+            return self._extract_single_field(element, field)
+        except Exception as e:
+            if self.verbose:
+                print(f"Error extracting field {field['name']}: {str(e)}")
+            return field.get('default')
+
+    def _extract_list_item(self, element, fields):
+        item = {}
+        for field in fields:
+            value = self._extract_single_field(element, field)
+            if value is not None:
+                item[field['name']] = value
+        return item
+    
+    def _extract_single_field(self, element, field):
+        if 'selector' in field:
+            selected = element.select_one(field['selector'])
+            if not selected:
+                return field.get('default')
+        else:
+            selected = element
+
+        value = None
+        if field['type'] == 'text':
+            value = selected.get_text(strip=True)
+        elif field['type'] == 'attribute':
+            value = selected.get(field['attribute'])
+        elif field['type'] == 'html':
+            value = str(selected)
+        elif field['type'] == 'regex':
+            text = selected.get_text(strip=True)
+            match = re.search(field['pattern'], text)
+            value = match.group(1) if match else None
+
+        if 'transform' in field:
+            value = self._apply_transform(value, field['transform'])
+
+        return value if value is not None else field.get('default')
+
+    def _extract_item(self, element, fields):
+        item = {}
+        for field in fields:
+            if field['type'] == 'computed':
+                value = self._compute_field(item, field)
+            else:
+                value = self._extract_field(element, field)
+            if value is not None:
+                item[field['name']] = value
+        return item
+    
+    def _apply_transform(self, value, transform):
+        if transform == 'lowercase':
+            return value.lower()
+        elif transform == 'uppercase':
+            return value.upper()
+        elif transform == 'strip':
+            return value.strip()
+        return value
+
+    def _compute_field(self, item, field):
+        try:
+            if 'expression' in field:
+                return eval(field['expression'], {}, item)
+            elif 'function' in field:
+                return field['function'](item)
+        except Exception as e:
+            if self.verbose:
+                print(f"Error computing field {field['name']}: {str(e)}")
+            return field.get('default')
+
+    def run(self, url: str, sections: List[str], *q, **kwargs) -> List[Dict[str, Any]]:
+        combined_html = self.DEL.join(sections)
+        return self.extract(url, combined_html, **kwargs)
+class _JsonXPathExtractionStrategy(ExtractionStrategy):
+    def __init__(self, schema: Dict[str, Any], **kwargs):
+        kwargs['input_format'] = 'html'  # Force HTML input
+        super().__init__(**kwargs)
+        self.schema = schema
+
+    def extract(self, url: str, html_content: str, *q, **kwargs) -> List[Dict[str, Any]]:
+        tree = html.fromstring(html_content)
+        base_xpath = self.schema['baseSelector']
+        base_elements = tree.xpath(base_xpath)
+        
+        results = []
+        for element in base_elements:
+            # Extract base element attributes first
+            item = {}
+            if 'baseFields' in self.schema:
+                for field in self.schema['baseFields']:
+                    value = self._extract_single_field(element, field)
+                    if value is not None:
+                        item[field['name']] = value
+            
+            # Then extract child fields
+            field_data = self._extract_item(element, self.schema['fields'])
+            item.update(field_data)
+            
+            results.append(item)
+        
+        return results
+
+    def _css_to_xpath(self, css_selector: str) -> str:
+        """Convert CSS selector to XPath if needed"""
+        if '/' in css_selector:  # Already an XPath
+            return css_selector
+        else:
+            # Fallback to basic conversion for common cases
+            return self._basic_css_to_xpath(css_selector)
+
+    def _basic_css_to_xpath(self, css_selector: str) -> str:
+        """Basic CSS to XPath conversion for common cases"""
+        # Handle basic cases
+        if ' > ' in css_selector:
+            parts = css_selector.split(' > ')
+            return '//' + '/'.join(parts)
+        if ' ' in css_selector:
+            parts = css_selector.split(' ')
+            return '//' + '//'.join(parts)
+        return '//' + css_selector
+
+    def _extract_field(self, element, field):
+        try:
+            if field['type'] == 'nested':
+                xpath = self._css_to_xpath(field['selector'])
+                nested_element = element.xpath(xpath)[0] if element.xpath(xpath) else None
+                return self._extract_item(nested_element, field['fields']) if nested_element is not None else {}
+            
+            if field['type'] == 'list':
+                xpath = self._css_to_xpath(field['selector'])
+                elements = element.xpath(xpath)
+                return [self._extract_list_item(el, field['fields']) for el in elements]
+            
+            if field['type'] == 'nested_list':
+                xpath = self._css_to_xpath(field['selector'])
+                elements = element.xpath(xpath)
+                return [self._extract_item(el, field['fields']) for el in elements]
+            
+            return self._extract_single_field(element, field)
+        except Exception as e:
+            if self.verbose:
+                print(f"Error extracting field {field['name']}: {str(e)}")
+            return field.get('default')
+
+    def _extract_list_item(self, element, fields):
+        item = {}
+        for field in fields:
+            value = self._extract_single_field(element, field)
+            if value is not None:
+                item[field['name']] = value
+        return item
+    
+    def _extract_single_field(self, element, field):
+        if 'selector' in field:
+            xpath = self._css_to_xpath(field['selector'])
+            selected = element.xpath(xpath)
+            if not selected:
+                return field.get('default')
+            selected = selected[0]
+        else:
+            selected = element
+
+        value = None
+        if field['type'] == 'text':
+            value = ''.join(selected.xpath('.//text()')).strip()
+        elif field['type'] == 'attribute':
+            value = selected.get(field['attribute'])
+        elif field['type'] == 'html':
+            value = etree.tostring(selected, encoding='unicode')
+        elif field['type'] == 'regex':
+            text = ''.join(selected.xpath('.//text()')).strip()
+            match = re.search(field['pattern'], text)
+            value = match.group(1) if match else None
+
+        if 'transform' in field:
+            value = self._apply_transform(value, field['transform'])
+
+        return value if value is not None else field.get('default')
+
+    def _extract_item(self, element, fields):
+        item = {}
+        for field in fields:
+            if field['type'] == 'computed':
+                value = self._compute_field(item, field)
+            else:
+                value = self._extract_field(element, field)
+            if value is not None:
+                item[field['name']] = value
+        return item
+    
+    def _apply_transform(self, value, transform):
+        if transform == 'lowercase':
+            return value.lower()
+        elif transform == 'uppercase':
+            return value.upper()
+        elif transform == 'strip':
+            return value.strip()
+        return value
+
+    def _compute_field(self, item, field):
+        try:
+            if 'expression' in field:
+                return eval(field['expression'], {}, item)
+            elif 'function' in field:
+                return field['function'](item)
+        except Exception as e:
+            if self.verbose:
+                print(f"Error computing field {field['name']}: {str(e)}")
+            return field.get('default')
+
+    def run(self, url: str, sections: List[str], *q, **kwargs) -> List[Dict[str, Any]]:
+        combined_html = self.DEL.join(sections)
+        return self.extract(url, combined_html, **kwargs)    

crawl4ai/extraction_strategy.py

@@ -0,0 +1,1052 @@
+from abc import ABC, abstractmethod
+from typing import Any, List, Dict, Optional, Union
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import json, time
+# from optimum.intel import IPEXModel
+from .prompts import *
+from .config import *
+from .utils import *
+from .models import *
+from functools import partial
+from .model_loader import *
+import math
+import numpy as np
+import re
+from bs4 import BeautifulSoup
+from lxml import html, etree
+from dataclasses import dataclass
+
+class ExtractionStrategy(ABC):
+    """
+    Abstract base class for all extraction strategies.
+    """
+    
+    def __init__(self, input_format: str = "markdown", **kwargs):
+        """
+        Initialize the extraction strategy.
+
+        Args:
+            input_format: Content format to use for extraction.
+                         Options: "markdown" (default), "html", "fit_markdown"
+            **kwargs: Additional keyword arguments
+        """
+        self.input_format = input_format
+        self.DEL = "<|DEL|>"
+        self.name = self.__class__.__name__
+        self.verbose = kwargs.get("verbose", False)
+
+    @abstractmethod
+    def extract(self, url: str, html: str, *q, **kwargs) -> List[Dict[str, Any]]:
+        """
+        Extract meaningful blocks or chunks from the given HTML.
+
+        :param url: The URL of the webpage.
+        :param html: The HTML content of the webpage.
+        :return: A list of extracted blocks or chunks.
+        """
+        pass
+    
+    def run(self, url: str, sections: List[str], *q, **kwargs) -> List[Dict[str, Any]]:
+        """
+        Process sections of text in parallel by default.
+
+        :param url: The URL of the webpage.
+        :param sections: List of sections (strings) to process.
+        :return: A list of processed JSON blocks.
+        """
+        extracted_content = []
+        with ThreadPoolExecutor() as executor:
+            futures = [executor.submit(self.extract, url, section, **kwargs) for section in sections]
+            for future in as_completed(futures):
+                extracted_content.extend(future.result())
+        return extracted_content    
+    
+class NoExtractionStrategy(ExtractionStrategy):
+    """
+    A strategy that does not extract any meaningful content from the HTML. It simply returns the entire HTML as a single block.
+    """
+    def extract(self, url: str, html: str, *q, **kwargs) -> List[Dict[str, Any]]:
+        """
+        Extract meaningful blocks or chunks from the given HTML.
+        """
+        return [{"index": 0, "content": html}]
+    
+    def run(self, url: str, sections: List[str], *q, **kwargs) -> List[Dict[str, Any]]:
+        return [{"index": i, "tags": [], "content": section} for i, section in enumerate(sections)]
+
+#######################################################
+# Strategies using LLM-based extraction for text data #
+#######################################################
+class LLMExtractionStrategy(ExtractionStrategy):
+    """
+    A strategy that uses an LLM to extract meaningful content from the HTML.
+    
+    Attributes:
+        provider: The provider to use for extraction. It follows the format <provider_name>/<model_name>, e.g., "ollama/llama3.3".
+        api_token: The API token for the provider.
+        instruction: The instruction to use for the LLM model.  
+        schema: Pydantic model schema for structured data.
+        extraction_type: "block" or "schema".
+        chunk_token_threshold: Maximum tokens per chunk.
+        overlap_rate: Overlap between chunks.
+        word_token_rate: Word to token conversion rate.
+        apply_chunking: Whether to apply chunking.
+        base_url: The base URL for the API request.
+        api_base: The base URL for the API request.
+        extra_args: Additional arguments for the API request, such as temprature, max_tokens, etc.
+        verbose: Whether to print verbose output.
+        usages: List of individual token usages.
+        total_usage: Accumulated token usage.
+    """
+
+    def __init__(self, 
+                 provider: str = DEFAULT_PROVIDER, api_token: Optional[str] = None, 
+                 instruction:str = None, schema:Dict = None, extraction_type = "block", **kwargs):
+        """
+        Initialize the strategy with clustering parameters.
+        
+        Args:
+            provider: The provider to use for extraction. It follows the format <provider_name>/<model_name>, e.g., "ollama/llama3.3".
+            api_token: The API token for the provider.
+            instruction: The instruction to use for the LLM model.  
+            schema: Pydantic model schema for structured data.
+            extraction_type: "block" or "schema".
+            chunk_token_threshold: Maximum tokens per chunk.
+            overlap_rate: Overlap between chunks.
+            word_token_rate: Word to token conversion rate.
+            apply_chunking: Whether to apply chunking.
+            base_url: The base URL for the API request.
+            api_base: The base URL for the API request.
+            extra_args: Additional arguments for the API request, such as temprature, max_tokens, etc.
+            verbose: Whether to print verbose output.
+            usages: List of individual token usages.
+            total_usage: Accumulated token usage.   
+
+        """
+        super().__init__(**kwargs)
+        self.provider = provider
+        self.api_token = api_token or PROVIDER_MODELS.get(provider, "no-token") or os.getenv("OPENAI_API_KEY")
+        self.instruction = instruction
+        self.extract_type = extraction_type
+        self.schema = schema
+        if schema:
+            self.extract_type = "schema"
+        
+        self.chunk_token_threshold = kwargs.get("chunk_token_threshold", CHUNK_TOKEN_THRESHOLD)
+        self.overlap_rate = kwargs.get("overlap_rate", OVERLAP_RATE)
+        self.word_token_rate = kwargs.get("word_token_rate", WORD_TOKEN_RATE)
+        self.apply_chunking = kwargs.get("apply_chunking", True)
+        self.base_url = kwargs.get("base_url", None)
+        self.api_base = kwargs.get("api_base", kwargs.get("base_url", None))
+        self.extra_args = kwargs.get("extra_args", {})
+        if not self.apply_chunking:
+            self.chunk_token_threshold = 1e9
+        
+        self.verbose = kwargs.get("verbose", False)
+        self.usages = []  # Store individual usages
+        self.total_usage = TokenUsage()  # Accumulated usage        
+        
+        if not self.api_token:
+            raise ValueError("API token must be provided for LLMExtractionStrategy. Update the config.py or set OPENAI_API_KEY environment variable.")
+        
+            
+    def extract(self, url: str, ix:int, html: str) -> List[Dict[str, Any]]:
+        """
+        Extract meaningful blocks or chunks from the given HTML using an LLM.
+        
+        How it works:
+        1. Construct a prompt with variables.
+        2. Make a request to the LLM using the prompt.
+        3. Parse the response and extract blocks or chunks.
+        
+        Args:
+            url: The URL of the webpage.
+            ix: Index of the block.
+            html: The HTML content of the webpage.
+            
+        Returns:
+            A list of extracted blocks or chunks.
+        """
+        if self.verbose:
+            # print("[LOG] Extracting blocks from URL:", url)
+            print(f"[LOG] Call LLM for {url} - block index: {ix}")
+
+        variable_values = {
+            "URL": url,
+            "HTML": escape_json_string(sanitize_html(html)),
+        }
+        
+        prompt_with_variables = PROMPT_EXTRACT_BLOCKS
+        if self.instruction:
+            variable_values["REQUEST"] = self.instruction
+            prompt_with_variables = PROMPT_EXTRACT_BLOCKS_WITH_INSTRUCTION
+            
+        if self.extract_type == "schema" and self.schema:
+            variable_values["SCHEMA"] = json.dumps(self.schema, indent=2)
+            prompt_with_variables = PROMPT_EXTRACT_SCHEMA_WITH_INSTRUCTION
+
+        for variable in variable_values:
+            prompt_with_variables = prompt_with_variables.replace(
+                "{" + variable + "}", variable_values[variable]
+            )
+        
+        response = perform_completion_with_backoff(
+            self.provider, 
+            prompt_with_variables, 
+            self.api_token, 
+            base_url=self.api_base or self.base_url,
+            extra_args = self.extra_args
+            ) # , json_response=self.extract_type == "schema")
+        # Track usage
+        usage = TokenUsage(
+            completion_tokens=response.usage.completion_tokens,
+            prompt_tokens=response.usage.prompt_tokens,
+            total_tokens=response.usage.total_tokens,
+            completion_tokens_details=response.usage.completion_tokens_details.__dict__ if response.usage.completion_tokens_details else {},
+            prompt_tokens_details=response.usage.prompt_tokens_details.__dict__ if response.usage.prompt_tokens_details else {}
+        )
+        self.usages.append(usage)
+        
+        # Update totals
+        self.total_usage.completion_tokens += usage.completion_tokens
+        self.total_usage.prompt_tokens += usage.prompt_tokens 
+        self.total_usage.total_tokens += usage.total_tokens
+        
+        try:
+            blocks = extract_xml_data(["blocks"], response.choices[0].message.content)['blocks']
+            blocks = json.loads(blocks)
+            for block in blocks:
+                block['error'] = False
+        except Exception as e:
+            parsed, unparsed = split_and_parse_json_objects(response.choices[0].message.content)
+            blocks = parsed
+            if unparsed:
+                blocks.append({
+                    "index": 0,
+                    "error": True,
+                    "tags": ["error"],
+                    "content": unparsed
+                })
+        
+        if self.verbose:
+            print("[LOG] Extracted", len(blocks), "blocks from URL:", url, "block index:", ix)
+        return blocks
+    
+    def _merge(self, documents, chunk_token_threshold, overlap):
+        """
+        Merge documents into sections based on chunk_token_threshold and overlap.
+        """
+        chunks = []
+        sections = []
+        total_tokens = 0
+
+        # Calculate the total tokens across all documents
+        for document in documents:
+            total_tokens += len(document.split(' ')) * self.word_token_rate
+
+        # Calculate the number of sections needed
+        num_sections = math.floor(total_tokens / chunk_token_threshold)
+        if num_sections < 1:
+            num_sections = 1  # Ensure there is at least one section
+        adjusted_chunk_threshold = total_tokens / num_sections
+
+        total_token_so_far = 0
+        current_chunk = []
+
+        for document in documents:
+            tokens = document.split(' ')
+            token_count = len(tokens) * self.word_token_rate
+            
+            if total_token_so_far + token_count <= adjusted_chunk_threshold:
+                current_chunk.extend(tokens)
+                total_token_so_far += token_count
+            else:
+                # Ensure to handle the last section properly
+                if len(sections) == num_sections - 1:
+                    current_chunk.extend(tokens)
+                    continue
+                
+                # Add overlap if specified
+                if overlap > 0 and current_chunk:
+                    overlap_tokens = current_chunk[-overlap:]
+                    current_chunk.extend(overlap_tokens)
+                
+                sections.append(' '.join(current_chunk))
+                current_chunk = tokens
+                total_token_so_far = token_count
+
+        # Add the last chunk
+        if current_chunk:
+            sections.append(' '.join(current_chunk))
+
+        return sections
+
+
+    def run(self, url: str, sections: List[str]) -> List[Dict[str, Any]]:
+        """
+        Process sections sequentially with a delay for rate limiting issues, specifically for LLMExtractionStrategy.
+        
+        Args:
+            url: The URL of the webpage.
+            sections: List of sections (strings) to process.
+            
+        Returns:
+            A list of extracted blocks or chunks.
+        """
+        
+        merged_sections = self._merge(
+            sections, self.chunk_token_threshold,
+            overlap= int(self.chunk_token_threshold * self.overlap_rate)
+        )
+        extracted_content = []
+        if self.provider.startswith("groq/"):
+            # Sequential processing with a delay
+            for ix, section in enumerate(merged_sections):
+                extract_func = partial(self.extract, url)
+                extracted_content.extend(extract_func(ix, sanitize_input_encode(section)))
+                time.sleep(0.5)  # 500 ms delay between each processing
+        else:
+            # Parallel processing using ThreadPoolExecutor
+            # extract_func = partial(self.extract, url)
+            # for ix, section in enumerate(merged_sections):
+            #     extracted_content.append(extract_func(ix, section))            
+            
+            with ThreadPoolExecutor(max_workers=4) as executor:
+                extract_func = partial(self.extract, url)
+                futures = [executor.submit(extract_func, ix, sanitize_input_encode(section)) for ix, section in enumerate(merged_sections)]
+                
+                for future in as_completed(futures):
+                    try:
+                        extracted_content.extend(future.result())
+                    except Exception as e:
+                        if self.verbose:
+                            print(f"Error in thread execution: {e}")
+                        # Add error information to extracted_content
+                        extracted_content.append({
+                            "index": 0,
+                            "error": True,
+                            "tags": ["error"],
+                            "content": str(e)
+                        })
+
+        
+        return extracted_content        
+    
+    
+    def show_usage(self) -> None:
+        """Print a detailed token usage report showing total and per-request usage."""
+        print("\n=== Token Usage Summary ===")
+        print(f"{'Type':<15} {'Count':>12}")
+        print("-" * 30)
+        print(f"{'Completion':<15} {self.total_usage.completion_tokens:>12,}")
+        print(f"{'Prompt':<15} {self.total_usage.prompt_tokens:>12,}")
+        print(f"{'Total':<15} {self.total_usage.total_tokens:>12,}")
+
+        print("\n=== Usage History ===")
+        print(f"{'Request #':<10} {'Completion':>12} {'Prompt':>12} {'Total':>12}")
+        print("-" * 48)
+        for i, usage in enumerate(self.usages, 1):
+            print(f"{i:<10} {usage.completion_tokens:>12,} {usage.prompt_tokens:>12,} {usage.total_tokens:>12,}")
+  
+#######################################################
+# Strategies using clustering for text data extraction #
+#######################################################
+
+class CosineStrategy(ExtractionStrategy):
+    """
+    Extract meaningful blocks or chunks from the given HTML using cosine similarity.
+    
+    How it works:
+    1. Pre-filter documents using embeddings and semantic_filter.
+    2. Perform clustering using cosine similarity.
+    3. Organize texts by their cluster labels, retaining order.
+    4. Filter clusters by word count.
+    5. Extract meaningful blocks or chunks from the filtered clusters.
+    
+    Attributes:
+        semantic_filter (str): A keyword filter for document filtering.
+        word_count_threshold (int): Minimum number of words per cluster.
+        max_dist (float): The maximum cophenetic distance on the dendrogram to form clusters.
+        linkage_method (str): The linkage method for hierarchical clustering.
+        top_k (int): Number of top categories to extract.
+        model_name (str): The name of the sentence-transformers model.
+        sim_threshold (float): The similarity threshold for clustering.
+    """ 
+    def __init__(self, semantic_filter = None, word_count_threshold=10, max_dist=0.2, linkage_method='ward', top_k=3, model_name = 'sentence-transformers/all-MiniLM-L6-v2', sim_threshold = 0.3, **kwargs):
+        """
+        Initialize the strategy with clustering parameters.
+
+        Args:
+            semantic_filter (str): A keyword filter for document filtering.
+            word_count_threshold (int): Minimum number of words per cluster.
+            max_dist (float): The maximum cophenetic distance on the dendrogram to form clusters.
+            linkage_method (str): The linkage method for hierarchical clustering.
+            top_k (int): Number of top categories to extract.
+        """
+        super().__init__(**kwargs)
+        
+        import numpy as np
+        
+        self.semantic_filter = semantic_filter
+        self.word_count_threshold = word_count_threshold
+        self.max_dist = max_dist
+        self.linkage_method = linkage_method
+        self.top_k = top_k
+        self.sim_threshold = sim_threshold
+        self.timer = time.time()
+        self.verbose = kwargs.get("verbose", False)
+        
+        self.buffer_embeddings = np.array([])
+        self.get_embedding_method = "direct"
+        
+        self.device = get_device()
+        # import torch
+        # self.device = torch.device('cpu')
+        
+        self.default_batch_size = calculate_batch_size(self.device)
+
+        if self.verbose:
+            print(f"[LOG] Loading Extraction Model for {self.device.type} device.")
+
+        # if False and self.device.type == "cpu":
+        #     self.model = load_onnx_all_MiniLM_l6_v2()
+        #     self.tokenizer = self.model.tokenizer
+        #     self.get_embedding_method = "direct"
+        # else:
+
+        self.tokenizer, self.model = load_HF_embedding_model(model_name)
+        self.model.to(self.device)
+        self.model.eval()  
+        
+        self.get_embedding_method = "batch"
+        
+        self.buffer_embeddings = np.array([])
+
+        # if model_name == "bert-base-uncased":
+        #     self.tokenizer, self.model = load_bert_base_uncased()
+        #     self.model.eval()  # Ensure the model is in evaluation mode
+        #     self.get_embedding_method = "batch"
+        # elif model_name == "BAAI/bge-small-en-v1.5":
+        #     self.tokenizer, self.model = load_bge_small_en_v1_5()
+        #     self.model.eval()  # Ensure the model is in evaluation mode
+        #     self.get_embedding_method = "batch"
+        # elif model_name == "sentence-transformers/all-MiniLM-L6-v2":
+        #     self.model = load_onnx_all_MiniLM_l6_v2()
+        #     self.tokenizer = self.model.tokenizer
+        #     self.get_embedding_method = "direct"
+       
+        
+        if self.verbose:
+            print(f"[LOG] Loading Multilabel Classifier for {self.device.type} device.")
+            
+        self.nlp, _ = load_text_multilabel_classifier()
+        # self.default_batch_size = 16 if self.device.type == 'cpu' else 64
+        
+        if self.verbose:
+            print(f"[LOG] Model loaded {model_name}, models/reuters, took " + str(time.time() - self.timer) + " seconds")
+
+    def filter_documents_embeddings(self, documents: List[str], semantic_filter: str, at_least_k: int = 20) -> List[str]:
+        """
+        Filter and sort documents based on the cosine similarity of their embeddings with the semantic_filter embedding.
+
+        Args:
+            documents (List[str]): A list of document texts.
+            semantic_filter (str): A keyword filter for document filtering.
+            at_least_k (int): The minimum number of documents to return.
+
+        Returns:
+            List[str]: A list of filtered and sorted document texts.
+        """
+        
+        if not semantic_filter:
+            return documents
+        
+        if len(documents) < at_least_k:
+            at_least_k = len(documents) // 2
+        
+        from sklearn.metrics.pairwise import cosine_similarity
+        
+        # Compute embedding for the keyword filter
+        query_embedding = self.get_embeddings([semantic_filter])[0]
+        
+        # Compute embeddings for the documents
+        document_embeddings = self.get_embeddings(documents)
+        
+        # Calculate cosine similarity between the query embedding and document embeddings
+        similarities = cosine_similarity([query_embedding], document_embeddings).flatten()
+        
+        # Filter documents based on the similarity threshold
+        filtered_docs = [(doc, sim) for doc, sim in zip(documents, similarities) if sim >= self.sim_threshold]
+        
+        # If the number of filtered documents is less than at_least_k, sort remaining documents by similarity
+        if len(filtered_docs) < at_least_k:
+            remaining_docs = [(doc, sim) for doc, sim in zip(documents, similarities) if sim < self.sim_threshold]
+            remaining_docs.sort(key=lambda x: x[1], reverse=True)
+            filtered_docs.extend(remaining_docs[:at_least_k - len(filtered_docs)])
+        
+        # Extract the document texts from the tuples
+        filtered_docs = [doc for doc, _ in filtered_docs]
+        
+        return filtered_docs[:at_least_k]
+    
+    def get_embeddings(self, sentences: List[str], batch_size=None, bypass_buffer=False):
+        """
+        Get BERT embeddings for a list of sentences.
+
+        Args:
+            sentences (List[str]): A list of text chunks (sentences).
+
+        Returns:
+            NumPy array of embeddings.
+        """
+        # if self.buffer_embeddings.any() and not bypass_buffer:
+        #     return self.buffer_embeddings
+        
+        if self.device.type in [ "cpu", "gpu", "cuda", "mps"]:
+            import torch 
+            # Tokenize sentences and convert to tensor
+            if batch_size is None:
+                batch_size = self.default_batch_size
+                        
+            all_embeddings = []
+            for i in range(0, len(sentences), batch_size):
+                batch_sentences = sentences[i:i + batch_size]
+                encoded_input = self.tokenizer(batch_sentences, padding=True, truncation=True, return_tensors='pt')
+                encoded_input = {key: tensor.to(self.device) for key, tensor in encoded_input.items()}
+                
+                # Ensure no gradients are calculated
+                with torch.no_grad():
+                    model_output = self.model(**encoded_input)
+                
+                # Get embeddings from the last hidden state (mean pooling)
+                embeddings = model_output.last_hidden_state.mean(dim=1).cpu().numpy()
+                all_embeddings.append(embeddings)
+            
+            self.buffer_embeddings = np.vstack(all_embeddings)
+        elif self.device.type == "cpu":      
+            # self.buffer_embeddings = self.model(sentences)
+            if batch_size is None:
+                batch_size = self.default_batch_size
+                
+            all_embeddings = []
+            for i in range(0, len(sentences), batch_size):
+                batch_sentences = sentences[i:i + batch_size]
+                embeddings = self.model(batch_sentences)
+                all_embeddings.append(embeddings)
+                
+            self.buffer_embeddings = np.vstack(all_embeddings)
+        return self.buffer_embeddings
+
+    def hierarchical_clustering(self, sentences: List[str], embeddings = None):
+        """
+        Perform hierarchical clustering on sentences and return cluster labels.
+
+        Args:
+            sentences (List[str]): A list of text chunks (sentences).
+
+        Returns:
+            NumPy array of cluster labels.
+        """
+        # Get embeddings
+        from scipy.cluster.hierarchy import linkage, fcluster
+        from scipy.spatial.distance import pdist
+        self.timer = time.time()
+        embeddings = self.get_embeddings(sentences, bypass_buffer=True)
+        # print(f"[LOG] 🚀 Embeddings computed in {time.time() - self.timer:.2f} seconds")
+        # Compute pairwise cosine distances
+        distance_matrix = pdist(embeddings, 'cosine')
+        # Perform agglomerative clustering respecting order
+        linked = linkage(distance_matrix, method=self.linkage_method)
+        # Form flat clusters
+        labels = fcluster(linked, self.max_dist, criterion='distance')
+        return labels
+
+    def filter_clusters_by_word_count(self, clusters: Dict[int, List[str]]) -> Dict[int, List[str]]:
+        """
+        Filter clusters to remove those with a word count below the threshold.
+
+        Args:
+            clusters (Dict[int, List[str]]): Dictionary of clusters.
+
+        Returns:
+            Dict[int, List[str]]: Filtered dictionary of clusters.
+        """
+        filtered_clusters = {}
+        for cluster_id, texts in clusters.items():
+            # Concatenate texts for analysis
+            full_text = " ".join(texts)
+            # Count words
+            word_count = len(full_text.split())
+            
+            # Keep clusters with word count above the threshold
+            if word_count >= self.word_count_threshold:
+                filtered_clusters[cluster_id] = texts
+
+        return filtered_clusters
+
+    def extract(self, url: str, html: str, *q, **kwargs) -> List[Dict[str, Any]]:
+        """
+        Extract clusters from HTML content using hierarchical clustering.
+
+        Args:
+            url (str): The URL of the webpage.
+            html (str): The HTML content of the webpage.
+
+        Returns:
+            List[Dict[str, Any]]: A list of processed JSON blocks.
+        """
+        # Assume `html` is a list of text chunks for this strategy
+        t = time.time()
+        text_chunks = html.split(self.DEL)  # Split by lines or paragraphs as needed
+        
+        # Pre-filter documents using embeddings and semantic_filter
+        text_chunks = self.filter_documents_embeddings(text_chunks, self.semantic_filter)
+
+        if not text_chunks:
+            return []
+
+        # Perform clustering
+        labels = self.hierarchical_clustering(text_chunks)
+        # print(f"[LOG] 🚀 Clustering done in {time.time() - t:.2f} seconds")
+
+        # Organize texts by their cluster labels, retaining order
+        t = time.time()
+        clusters = {}
+        for index, label in enumerate(labels):
+            clusters.setdefault(label, []).append(text_chunks[index])
+
+        # Filter clusters by word count
+        filtered_clusters = self.filter_clusters_by_word_count(clusters)
+
+        # Convert filtered clusters to a sorted list of dictionaries
+        cluster_list = [{"index": int(idx), "tags" : [], "content": " ".join(filtered_clusters[idx])} for idx in sorted(filtered_clusters)]
+        
+        if self.verbose:
+            print(f"[LOG] 🚀 Assign tags using {self.device}")
+        
+        if self.device.type in ["gpu", "cuda", "mps", "cpu"]:
+            labels = self.nlp([cluster['content'] for cluster in cluster_list])
+            
+            for cluster, label in zip(cluster_list, labels):
+                cluster['tags'] = label
+        # elif self.device.type == "cpu":
+        #     # Process the text with the loaded model
+        #     texts = [cluster['content'] for cluster in cluster_list]
+        #     # Batch process texts
+        #     docs = self.nlp.pipe(texts, disable=["tagger", "parser", "ner", "lemmatizer"])
+
+        #     for doc, cluster in zip(docs, cluster_list):
+        #         tok_k = self.top_k
+        #         top_categories = sorted(doc.cats.items(), key=lambda x: x[1], reverse=True)[:tok_k]
+        #         cluster['tags'] = [cat for cat, _ in top_categories]
+                            
+            # for cluster in  cluster_list:
+            #     doc = self.nlp(cluster['content'])
+            #     tok_k = self.top_k
+            #     top_categories = sorted(doc.cats.items(), key=lambda x: x[1], reverse=True)[:tok_k]
+            #     cluster['tags'] = [cat for cat, _ in top_categories]
+        
+        if self.verbose:
+            print(f"[LOG] 🚀 Categorization done in {time.time() - t:.2f} seconds")
+        
+        return cluster_list
+
+    def run(self, url: str, sections: List[str], *q, **kwargs) -> List[Dict[str, Any]]:
+        """
+        Process sections using hierarchical clustering.
+
+        Args:
+            url (str): The URL of the webpage.
+            sections (List[str]): List of sections (strings) to process.
+
+        Returns:
+        """
+        # This strategy processes all sections together
+        
+        return self.extract(url, self.DEL.join(sections), **kwargs)
+    
+#######################################################
+# New extraction strategies for JSON-based extraction #
+####################################################### 
+
+class JsonElementExtractionStrategy(ExtractionStrategy):
+    """
+    Abstract base class for extracting structured JSON from HTML content.
+
+    How it works:
+    1. Parses HTML content using the `_parse_html` method.
+    2. Uses a schema to define base selectors, fields, and transformations.
+    3. Extracts data hierarchically, supporting nested fields and lists.
+    4. Handles computed fields with expressions or functions.
+
+    Attributes:
+        DEL (str): Delimiter used to combine HTML sections. Defaults to '\n'.
+        schema (Dict[str, Any]): The schema defining the extraction rules.
+        verbose (bool): Enables verbose logging for debugging purposes.
+
+    Methods:
+        extract(url, html_content, *q, **kwargs): Extracts structured data from HTML content.
+        _extract_item(element, fields): Extracts fields from a single element.
+        _extract_single_field(element, field): Extracts a single field based on its type.
+        _apply_transform(value, transform): Applies a transformation to a value.
+        _compute_field(item, field): Computes a field value using an expression or function.
+        run(url, sections, *q, **kwargs): Combines HTML sections and runs the extraction strategy.
+
+    Abstract Methods:
+        _parse_html(html_content): Parses raw HTML into a structured format (e.g., BeautifulSoup or lxml).
+        _get_base_elements(parsed_html, selector): Retrieves base elements using a selector.
+        _get_elements(element, selector): Retrieves child elements using a selector.
+        _get_element_text(element): Extracts text content from an element.
+        _get_element_html(element): Extracts raw HTML from an element.
+        _get_element_attribute(element, attribute): Extracts an attribute's value from an element.
+    """
+
+    
+    DEL = '\n'
+
+    def __init__(self, schema: Dict[str, Any], **kwargs):
+        """
+        Initialize the JSON element extraction strategy with a schema.
+
+        Args:
+            schema (Dict[str, Any]): The schema defining the extraction rules.
+        """
+        super().__init__(**kwargs)
+        self.schema = schema
+        self.verbose = kwargs.get('verbose', False)
+
+    def extract(self, url: str, html_content: str, *q, **kwargs) -> List[Dict[str, Any]]:
+        """
+        Extract structured data from HTML content.
+
+        How it works:
+        1. Parses the HTML content using the `_parse_html` method.
+        2. Identifies base elements using the schema's base selector.
+        3. Extracts fields from each base element using `_extract_item`.
+
+        Args:
+            url (str): The URL of the page being processed.
+            html_content (str): The raw HTML content to parse and extract.
+            *q: Additional positional arguments.
+            **kwargs: Additional keyword arguments for custom extraction.
+
+        Returns:
+            List[Dict[str, Any]]: A list of extracted items, each represented as a dictionary.
+        """
+        
+        parsed_html = self._parse_html(html_content)
+        base_elements = self._get_base_elements(parsed_html, self.schema['baseSelector'])
+        
+        results = []
+        for element in base_elements:
+            # Extract base element attributes
+            item = {}
+            if 'baseFields' in self.schema:
+                for field in self.schema['baseFields']:
+                    value = self._extract_single_field(element, field)
+                    if value is not None:
+                        item[field['name']] = value
+            
+            # Extract child fields
+            field_data = self._extract_item(element, self.schema['fields'])
+            item.update(field_data)
+            
+            if item:
+                results.append(item)
+        
+        return results
+
+    @abstractmethod
+    def _parse_html(self, html_content: str):
+        """Parse HTML content into appropriate format"""
+        pass
+
+    @abstractmethod
+    def _get_base_elements(self, parsed_html, selector: str):
+        """Get all base elements using the selector"""
+        pass
+
+    @abstractmethod
+    def _get_elements(self, element, selector: str):
+        """Get child elements using the selector"""
+        pass
+
+    def _extract_field(self, element, field):
+        try:
+            if field['type'] == 'nested':
+                nested_elements = self._get_elements(element, field['selector'])
+                nested_element = nested_elements[0] if nested_elements else None
+                return self._extract_item(nested_element, field['fields']) if nested_element else {}
+            
+            if field['type'] == 'list':
+                elements = self._get_elements(element, field['selector'])
+                return [self._extract_list_item(el, field['fields']) for el in elements]
+            
+            if field['type'] == 'nested_list':
+                elements = self._get_elements(element, field['selector'])
+                return [self._extract_item(el, field['fields']) for el in elements]
+            
+            return self._extract_single_field(element, field)
+        except Exception as e:
+            if self.verbose:
+                print(f"Error extracting field {field['name']}: {str(e)}")
+            return field.get('default')
+
+    def _extract_single_field(self, element, field):
+        """
+        Extract a single field based on its type.
+
+        How it works:
+        1. Selects the target element using the field's selector.
+        2. Extracts the field value based on its type (e.g., text, attribute, regex).
+        3. Applies transformations if defined in the schema.
+
+        Args:
+            element: The base element to extract the field from.
+            field (Dict[str, Any]): The field definition in the schema.
+
+        Returns:
+            Any: The extracted field value.
+        """
+        
+        if 'selector' in field:
+            selected = self._get_elements(element, field['selector'])
+            if not selected:
+                return field.get('default')
+            selected = selected[0]
+        else:
+            selected = element
+
+        value = None
+        if field['type'] == 'text':
+            value = self._get_element_text(selected)
+        elif field['type'] == 'attribute':
+            value = self._get_element_attribute(selected, field['attribute'])
+        elif field['type'] == 'html':
+            value = self._get_element_html(selected)
+        elif field['type'] == 'regex':
+            text = self._get_element_text(selected)
+            match = re.search(field['pattern'], text)
+            value = match.group(1) if match else None
+
+        if 'transform' in field:
+            value = self._apply_transform(value, field['transform'])
+
+        return value if value is not None else field.get('default')
+
+    def _extract_list_item(self, element, fields):
+        item = {}
+        for field in fields:
+            value = self._extract_single_field(element, field)
+            if value is not None:
+                item[field['name']] = value
+        return item
+
+    def _extract_item(self, element, fields):
+        """
+        Extracts fields from a given element.
+
+        How it works:
+        1. Iterates through the fields defined in the schema.
+        2. Handles computed, single, and nested field types.
+        3. Updates the item dictionary with extracted field values.
+
+        Args:
+            element: The base element to extract fields from.
+            fields (List[Dict[str, Any]]): The list of fields to extract.
+
+        Returns:
+            Dict[str, Any]: A dictionary representing the extracted item.
+        """
+        
+        item = {}
+        for field in fields:
+            if field['type'] == 'computed':
+                value = self._compute_field(item, field)
+            else:
+                value = self._extract_field(element, field)
+            if value is not None:
+                item[field['name']] = value
+        return item
+
+    def _apply_transform(self, value, transform):
+        """
+        Apply a transformation to a value.
+
+        How it works:
+        1. Checks the transformation type (e.g., `lowercase`, `strip`).
+        2. Applies the transformation to the value.
+        3. Returns the transformed value.
+
+        Args:
+            value (str): The value to transform.
+            transform (str): The type of transformation to apply.
+
+        Returns:
+            str: The transformed value.
+        """
+        
+        if transform == 'lowercase':
+            return value.lower()
+        elif transform == 'uppercase':
+            return value.upper()
+        elif transform == 'strip':
+            return value.strip()
+        return value
+
+    def _compute_field(self, item, field):
+        try:
+            if 'expression' in field:
+                return eval(field['expression'], {}, item)
+            elif 'function' in field:
+                return field['function'](item)
+        except Exception as e:
+            if self.verbose:
+                print(f"Error computing field {field['name']}: {str(e)}")
+            return field.get('default')
+
+    def run(self, url: str, sections: List[str], *q, **kwargs) -> List[Dict[str, Any]]:
+        """
+        Run the extraction strategy on a combined HTML content.
+
+        How it works:
+        1. Combines multiple HTML sections using the `DEL` delimiter.
+        2. Calls the `extract` method with the combined HTML.
+
+        Args:
+            url (str): The URL of the page being processed.
+            sections (List[str]): A list of HTML sections.
+            *q: Additional positional arguments.
+            **kwargs: Additional keyword arguments for custom extraction.
+
+        Returns:
+            List[Dict[str, Any]]: A list of extracted items.
+        """
+        
+        combined_html = self.DEL.join(sections)
+        return self.extract(url, combined_html, **kwargs)
+
+    @abstractmethod
+    def _get_element_text(self, element) -> str:
+        """Get text content from element"""
+        pass
+
+    @abstractmethod
+    def _get_element_html(self, element) -> str:
+        """Get HTML content from element"""
+        pass
+
+    @abstractmethod
+    def _get_element_attribute(self, element, attribute: str):
+        """Get attribute value from element"""
+        pass
+
+class JsonCssExtractionStrategy(JsonElementExtractionStrategy):
+    """
+    Concrete implementation of `JsonElementExtractionStrategy` using CSS selectors.
+
+    How it works:
+    1. Parses HTML content with BeautifulSoup.
+    2. Selects elements using CSS selectors defined in the schema.
+    3. Extracts field data and applies transformations as defined.
+
+    Attributes:
+        schema (Dict[str, Any]): The schema defining the extraction rules.
+        verbose (bool): Enables verbose logging for debugging purposes.
+
+    Methods:
+        _parse_html(html_content): Parses HTML content into a BeautifulSoup object.
+        _get_base_elements(parsed_html, selector): Selects base elements using a CSS selector.
+        _get_elements(element, selector): Selects child elements using a CSS selector.
+        _get_element_text(element): Extracts text content from a BeautifulSoup element.
+        _get_element_html(element): Extracts the raw HTML content of a BeautifulSoup element.
+        _get_element_attribute(element, attribute): Retrieves an attribute value from a BeautifulSoup element.
+    """
+    
+    def __init__(self, schema: Dict[str, Any], **kwargs):
+        kwargs['input_format'] = 'html'  # Force HTML input
+        super().__init__(schema, **kwargs)
+
+    def _parse_html(self, html_content: str):
+        return BeautifulSoup(html_content, 'html.parser')
+
+    def _get_base_elements(self, parsed_html, selector: str):
+        return parsed_html.select(selector)
+
+    def _get_elements(self, element, selector: str):
+        # Return all matching elements using select() instead of select_one()
+        # This ensures that we get all elements that match the selector, not just the first one
+        return element.select(selector)
+
+    def _get_element_text(self, element) -> str:
+        return element.get_text(strip=True)
+
+    def _get_element_html(self, element) -> str:
+        return str(element)
+
+    def _get_element_attribute(self, element, attribute: str):
+        return element.get(attribute)
+
+class JsonXPathExtractionStrategy(JsonElementExtractionStrategy):
+    """
+    Concrete implementation of `JsonElementExtractionStrategy` using XPath selectors.
+
+    How it works:
+    1. Parses HTML content into an lxml tree.
+    2. Selects elements using XPath expressions.
+    3. Converts CSS selectors to XPath when needed.
+
+    Attributes:
+        schema (Dict[str, Any]): The schema defining the extraction rules.
+        verbose (bool): Enables verbose logging for debugging purposes.
+
+    Methods:
+        _parse_html(html_content): Parses HTML content into an lxml tree.
+        _get_base_elements(parsed_html, selector): Selects base elements using an XPath selector.
+        _css_to_xpath(css_selector): Converts a CSS selector to an XPath expression.
+        _get_elements(element, selector): Selects child elements using an XPath selector.
+        _get_element_text(element): Extracts text content from an lxml element.
+        _get_element_html(element): Extracts the raw HTML content of an lxml element.
+        _get_element_attribute(element, attribute): Retrieves an attribute value from an lxml element.
+    """
+    
+    def __init__(self, schema: Dict[str, Any], **kwargs):
+        kwargs['input_format'] = 'html'  # Force HTML input
+        super().__init__(schema, **kwargs)
+
+    def _parse_html(self, html_content: str):
+        return html.fromstring(html_content)
+
+    def _get_base_elements(self, parsed_html, selector: str):
+        return parsed_html.xpath(selector)
+
+    def _css_to_xpath(self, css_selector: str) -> str:
+        """Convert CSS selector to XPath if needed"""
+        if '/' in css_selector:  # Already an XPath
+            return css_selector
+        return self._basic_css_to_xpath(css_selector)
+
+    def _basic_css_to_xpath(self, css_selector: str) -> str:
+        """Basic CSS to XPath conversion for common cases"""
+        if ' > ' in css_selector:
+            parts = css_selector.split(' > ')
+            return '//' + '/'.join(parts)
+        if ' ' in css_selector:
+            parts = css_selector.split(' ')
+            return '//' + '//'.join(parts)
+        return '//' + css_selector
+
+    def _get_elements(self, element, selector: str):
+        xpath = self._css_to_xpath(selector)
+        if not xpath.startswith('.'):
+            xpath = '.' + xpath
+        return element.xpath(xpath)
+
+    def _get_element_text(self, element) -> str:
+        return ''.join(element.xpath('.//text()')).strip()
+
+    def _get_element_html(self, element) -> str:
+        return etree.tostring(element, encoding='unicode')
+
+    def _get_element_attribute(self, element, attribute: str):
+        return element.get(attribute)

crawl4ai/html2text/__init__.py

@@ -0,0 +1,1141 @@
+"""html2text: Turn HTML into equivalent Markdown-structured text."""
+
+import html.entities
+import html.parser
+import re
+import string
+import urllib.parse as urlparse
+from textwrap import wrap
+from typing import Dict, List, Optional, Tuple, Union
+
+from . import config
+from ._typing import OutCallback
+from .elements import AnchorElement, ListElement
+from .utils import (
+    dumb_css_parser,
+    element_style,
+    escape_md,
+    escape_md_section,
+    google_fixed_width_font,
+    google_has_height,
+    google_list_style,
+    google_text_emphasis,
+    hn,
+    list_numbering_start,
+    pad_tables_in_text,
+    skipwrap,
+    unifiable_n,
+)
+
+__version__ = (2024, 2, 26)
+
+
+# TODO:
+# Support decoded entities with UNIFIABLE.
+
+
+class HTML2Text(html.parser.HTMLParser):
+    def __init__(
+        self,
+        out: Optional[OutCallback] = None,
+        baseurl: str = "",
+        bodywidth: int = config.BODY_WIDTH,
+    ) -> None:
+        """
+        Input parameters:
+            out: possible custom replacement for self.outtextf (which
+                 appends lines of text).
+            baseurl: base URL of the document we process
+        """
+        super().__init__(convert_charrefs=False)
+
+        # Config options
+        self.split_next_td = False
+        self.td_count = 0
+        self.table_start = False
+        self.unicode_snob = config.UNICODE_SNOB  # covered in cli
+        
+        self.escape_snob = config.ESCAPE_SNOB  # covered in cli
+        self.escape_backslash = config.ESCAPE_BACKSLASH  # covered in cli
+        self.escape_dot = config.ESCAPE_DOT  # covered in cli
+        self.escape_plus = config.ESCAPE_PLUS  # covered in cli
+        self.escape_dash = config.ESCAPE_DASH  # covered in cli
+        
+        self.links_each_paragraph = config.LINKS_EACH_PARAGRAPH
+        self.body_width = bodywidth  # covered in cli
+        self.skip_internal_links = config.SKIP_INTERNAL_LINKS  # covered in cli
+        self.inline_links = config.INLINE_LINKS  # covered in cli
+        self.protect_links = config.PROTECT_LINKS  # covered in cli
+        self.google_list_indent = config.GOOGLE_LIST_INDENT  # covered in cli
+        self.ignore_links = config.IGNORE_ANCHORS  # covered in cli
+        self.ignore_mailto_links = config.IGNORE_MAILTO_LINKS  # covered in cli
+        self.ignore_images = config.IGNORE_IMAGES  # covered in cli
+        self.images_as_html = config.IMAGES_AS_HTML  # covered in cli
+        self.images_to_alt = config.IMAGES_TO_ALT  # covered in cli
+        self.images_with_size = config.IMAGES_WITH_SIZE  # covered in cli
+        self.ignore_emphasis = config.IGNORE_EMPHASIS  # covered in cli
+        self.bypass_tables = config.BYPASS_TABLES  # covered in cli
+        self.ignore_tables = config.IGNORE_TABLES  # covered in cli
+        self.google_doc = False  # covered in cli
+        self.ul_item_mark = "*"  # covered in cli
+        self.emphasis_mark = "_"  # covered in cli
+        self.strong_mark = "**"
+        self.single_line_break = config.SINGLE_LINE_BREAK  # covered in cli
+        self.use_automatic_links = config.USE_AUTOMATIC_LINKS  # covered in cli
+        self.hide_strikethrough = False  # covered in cli
+        self.mark_code = config.MARK_CODE
+        self.wrap_list_items = config.WRAP_LIST_ITEMS  # covered in cli
+        self.wrap_links = config.WRAP_LINKS  # covered in cli
+        self.wrap_tables = config.WRAP_TABLES
+        self.pad_tables = config.PAD_TABLES  # covered in cli
+        self.default_image_alt = config.DEFAULT_IMAGE_ALT  # covered in cli
+        self.tag_callback = None
+        self.open_quote = config.OPEN_QUOTE  # covered in cli
+        self.close_quote = config.CLOSE_QUOTE  # covered in cli
+        self.include_sup_sub = config.INCLUDE_SUP_SUB  # covered in cli
+
+        if out is None:
+            self.out = self.outtextf
+        else:
+            self.out = out
+
+        # empty list to store output characters before they are "joined"
+        self.outtextlist: List[str] = []
+
+        self.quiet = 0
+        self.p_p = 0  # number of newline character to print before next output
+        self.outcount = 0
+        self.start = True
+        self.space = False
+        self.a: List[AnchorElement] = []
+        self.astack: List[Optional[Dict[str, Optional[str]]]] = []
+        self.maybe_automatic_link: Optional[str] = None
+        self.empty_link = False
+        self.absolute_url_matcher = re.compile(r"^[a-zA-Z+]+://")
+        self.acount = 0
+        self.list: List[ListElement] = []
+        self.blockquote = 0
+        self.pre = False
+        self.startpre = False
+        self.code = False
+        self.quote = False
+        self.br_toggle = ""
+        self.lastWasNL = False
+        self.lastWasList = False
+        self.style = 0
+        self.style_def: Dict[str, Dict[str, str]] = {}
+        self.tag_stack: List[Tuple[str, Dict[str, Optional[str]], Dict[str, str]]] = []
+        self.emphasis = 0
+        self.drop_white_space = 0
+        self.inheader = False
+        # Current abbreviation definition
+        self.abbr_title: Optional[str] = None
+        # Last inner HTML (for abbr being defined)
+        self.abbr_data: Optional[str] = None
+        # Stack of abbreviations to write later
+        self.abbr_list: Dict[str, str] = {}
+        self.baseurl = baseurl
+        self.stressed = False
+        self.preceding_stressed = False
+        self.preceding_data = ""
+        self.current_tag = ""
+
+        config.UNIFIABLE["nbsp"] = "&nbsp_place_holder;"
+
+    def update_params(self, **kwargs):
+        for key, value in kwargs.items():
+            setattr(self, key, value) 
+    
+    def feed(self, data: str) -> None:
+        data = data.replace("</' + 'script>", "</ignore>")
+        super().feed(data)
+
+    def handle(self, data: str) -> str:
+        self.start = True
+        self.feed(data)
+        self.feed("")
+        markdown = self.optwrap(self.finish())
+        if self.pad_tables:
+            return pad_tables_in_text(markdown)
+        else:
+            return markdown
+
+    def outtextf(self, s: str) -> None:
+        self.outtextlist.append(s)
+        if s:
+            self.lastWasNL = s[-1] == "\n"
+
+    def finish(self) -> str:
+        self.close()
+
+        self.pbr()
+        self.o("", force="end")
+
+        outtext = "".join(self.outtextlist)
+
+        if self.unicode_snob:
+            nbsp = html.entities.html5["nbsp;"]
+        else:
+            nbsp = " "
+        outtext = outtext.replace("&nbsp_place_holder;", nbsp)
+
+        # Clear self.outtextlist to avoid memory leak of its content to
+        # the next handling.
+        self.outtextlist = []
+
+        return outtext
+
+    def handle_charref(self, c: str) -> None:
+        self.handle_data(self.charref(c), True)
+
+    def handle_entityref(self, c: str) -> None:
+        ref = self.entityref(c)
+
+        # ref may be an empty string (e.g. for &lrm;/&rlm; markers that should
+        # not contribute to the final output).
+        # self.handle_data cannot handle a zero-length string right after a
+        # stressed tag or mid-text within a stressed tag (text get split and
+        # self.stressed/self.preceding_stressed gets switched after the first
+        # part of that text).
+        if ref:
+            self.handle_data(ref, True)
+
+    def handle_starttag(self, tag: str, attrs: List[Tuple[str, Optional[str]]]) -> None:
+        self.handle_tag(tag, dict(attrs), start=True)
+
+    def handle_endtag(self, tag: str) -> None:
+        self.handle_tag(tag, {}, start=False)
+
+    def previousIndex(self, attrs: Dict[str, Optional[str]]) -> Optional[int]:
+        """
+        :type attrs: dict
+
+        :returns: The index of certain set of attributes (of a link) in the
+        self.a list. If the set of attributes is not found, returns None
+        :rtype: int
+        """
+        if "href" not in attrs:
+            return None
+
+        match = False
+        for i, a in enumerate(self.a):
+            if "href" in a.attrs and a.attrs["href"] == attrs["href"]:
+                if "title" in a.attrs or "title" in attrs:
+                    if (
+                        "title" in a.attrs
+                        and "title" in attrs
+                        and a.attrs["title"] == attrs["title"]
+                    ):
+                        match = True
+                else:
+                    match = True
+
+            if match:
+                return i
+        return None
+
+    def handle_emphasis(
+        self, start: bool, tag_style: Dict[str, str], parent_style: Dict[str, str]
+    ) -> None:
+        """
+        Handles various text emphases
+        """
+        tag_emphasis = google_text_emphasis(tag_style)
+        parent_emphasis = google_text_emphasis(parent_style)
+
+        # handle Google's text emphasis
+        strikethrough = "line-through" in tag_emphasis and self.hide_strikethrough
+
+        # google and others may mark a font's weight as `bold` or `700`
+        bold = False
+        for bold_marker in config.BOLD_TEXT_STYLE_VALUES:
+            bold = bold_marker in tag_emphasis and bold_marker not in parent_emphasis
+            if bold:
+                break
+
+        italic = "italic" in tag_emphasis and "italic" not in parent_emphasis
+        fixed = (
+            google_fixed_width_font(tag_style)
+            and not google_fixed_width_font(parent_style)
+            and not self.pre
+        )
+
+        if start:
+            # crossed-out text must be handled before other attributes
+            # in order not to output qualifiers unnecessarily
+            if bold or italic or fixed:
+                self.emphasis += 1
+            if strikethrough:
+                self.quiet += 1
+            if italic:
+                self.o(self.emphasis_mark)
+                self.drop_white_space += 1
+            if bold:
+                self.o(self.strong_mark)
+                self.drop_white_space += 1
+            if fixed:
+                self.o("`")
+                self.drop_white_space += 1
+                self.code = True
+        else:
+            if bold or italic or fixed:
+                # there must not be whitespace before closing emphasis mark
+                self.emphasis -= 1
+                self.space = False
+            if fixed:
+                if self.drop_white_space:
+                    # empty emphasis, drop it
+                    self.drop_white_space -= 1
+                else:
+                    self.o("`")
+                self.code = False
+            if bold:
+                if self.drop_white_space:
+                    # empty emphasis, drop it
+                    self.drop_white_space -= 1
+                else:
+                    self.o(self.strong_mark)
+            if italic:
+                if self.drop_white_space:
+                    # empty emphasis, drop it
+                    self.drop_white_space -= 1
+                else:
+                    self.o(self.emphasis_mark)
+            # space is only allowed after *all* emphasis marks
+            if (bold or italic) and not self.emphasis:
+                self.o(" ")
+            if strikethrough:
+                self.quiet -= 1
+
+    def handle_tag(
+        self, tag: str, attrs: Dict[str, Optional[str]], start: bool
+    ) -> None:
+        self.current_tag = tag
+
+        if self.tag_callback is not None:
+            if self.tag_callback(self, tag, attrs, start) is True:
+                return
+
+        # first thing inside the anchor tag is another tag
+        # that produces some output
+        if (
+            start
+            and self.maybe_automatic_link is not None
+            and tag not in ["p", "div", "style", "dl", "dt"]
+            and (tag != "img" or self.ignore_images)
+        ):
+            self.o("[")
+            self.maybe_automatic_link = None
+            self.empty_link = False
+
+        if self.google_doc:
+            # the attrs parameter is empty for a closing tag. in addition, we
+            # need the attributes of the parent nodes in order to get a
+            # complete style description for the current element. we assume
+            # that google docs export well formed html.
+            parent_style: Dict[str, str] = {}
+            if start:
+                if self.tag_stack:
+                    parent_style = self.tag_stack[-1][2]
+                tag_style = element_style(attrs, self.style_def, parent_style)
+                self.tag_stack.append((tag, attrs, tag_style))
+            else:
+                dummy, attrs, tag_style = (
+                    self.tag_stack.pop() if self.tag_stack else (None, {}, {})
+                )
+                if self.tag_stack:
+                    parent_style = self.tag_stack[-1][2]
+
+        if hn(tag):
+            # check if nh is inside of an 'a' tag (incorrect but found in the wild)
+            if self.astack:
+                if start:
+                    self.inheader = True
+                    # are inside link name, so only add '#' if it can appear before '['
+                    if self.outtextlist and self.outtextlist[-1] == "[":
+                        self.outtextlist.pop()
+                        self.space = False
+                        self.o(hn(tag) * "#" + " ")
+                        self.o("[")
+                else:
+                    self.p_p = 0  # don't break up link name
+                    self.inheader = False
+                    return  # prevent redundant emphasis marks on headers
+            else:
+                self.p()
+                if start:
+                    self.inheader = True
+                    self.o(hn(tag) * "#" + " ")
+                else:
+                    self.inheader = False
+                    return  # prevent redundant emphasis marks on headers
+
+        if tag in ["p", "div"]:
+            if self.google_doc:
+                if start and google_has_height(tag_style):
+                    self.p()
+                else:
+                    self.soft_br()
+            elif self.astack:
+                pass
+            elif self.split_next_td:
+                pass
+            else:
+                self.p()
+
+        if tag == "br" and start:
+            if self.blockquote > 0:
+                self.o("  \n> ")
+            else:
+                self.o("  \n")
+
+        if tag == "hr" and start:
+            self.p()
+            self.o("* * *")
+            self.p()
+
+        if tag in ["head", "style", "script"]:
+            if start:
+                self.quiet += 1
+            else:
+                self.quiet -= 1
+
+        if tag == "style":
+            if start:
+                self.style += 1
+            else:
+                self.style -= 1
+
+        if tag in ["body"]:
+            self.quiet = 0  # sites like 9rules.com never close <head>
+
+        if tag == "blockquote":
+            if start:
+                self.p()
+                self.o("> ", force=True)
+                self.start = True
+                self.blockquote += 1
+            else:
+                self.blockquote -= 1
+                self.p()
+
+        if tag in ["em", "i", "u"] and not self.ignore_emphasis:
+            # Separate with a space if we immediately follow an alphanumeric
+            # character, since otherwise Markdown won't render the emphasis
+            # marks, and we'll be left with eg 'foo_bar_' visible.
+            # (Don't add a space otherwise, though, since there isn't one in the
+            # original HTML.)
+            if (
+                start
+                and self.preceding_data
+                and self.preceding_data[-1] not in string.whitespace
+                and self.preceding_data[-1] not in string.punctuation
+            ):
+                emphasis = " " + self.emphasis_mark
+                self.preceding_data += " "
+            else:
+                emphasis = self.emphasis_mark
+
+            self.o(emphasis)
+            if start:
+                self.stressed = True
+
+        if tag in ["strong", "b"] and not self.ignore_emphasis:
+            # Separate with space if we immediately follow an * character, since
+            # without it, Markdown won't render the resulting *** correctly.
+            # (Don't add a space otherwise, though, since there isn't one in the
+            # original HTML.)
+            if (
+                start
+                and self.preceding_data
+                # When `self.strong_mark` is set to empty, the next condition
+                # will cause IndexError since it's trying to match the data
+                # with the first character of the `self.strong_mark`.
+                and len(self.strong_mark) > 0
+                and self.preceding_data[-1] == self.strong_mark[0]
+            ):
+                strong = " " + self.strong_mark
+                self.preceding_data += " "
+            else:
+                strong = self.strong_mark
+
+            self.o(strong)
+            if start:
+                self.stressed = True
+
+        if tag in ["del", "strike", "s"]:
+            if start and self.preceding_data and self.preceding_data[-1] == "~":
+                strike = " ~~"
+                self.preceding_data += " "
+            else:
+                strike = "~~"
+
+            self.o(strike)
+            if start:
+                self.stressed = True
+
+        if self.google_doc:
+            if not self.inheader:
+                # handle some font attributes, but leave headers clean
+                self.handle_emphasis(start, tag_style, parent_style)
+
+        if tag in ["kbd", "code", "tt"] and not self.pre:
+            self.o("`")  # TODO: `` `this` ``
+            self.code = not self.code
+
+        if tag == "abbr":
+            if start:
+                self.abbr_title = None
+                self.abbr_data = ""
+                if "title" in attrs:
+                    self.abbr_title = attrs["title"]
+            else:
+                if self.abbr_title is not None:
+                    assert self.abbr_data is not None
+                    self.abbr_list[self.abbr_data] = self.abbr_title
+                    self.abbr_title = None
+                self.abbr_data = None
+
+        if tag == "q":
+            if not self.quote:
+                self.o(self.open_quote)
+            else:
+                self.o(self.close_quote)
+            self.quote = not self.quote
+
+        def link_url(self: HTML2Text, link: str, title: str = "") -> None:
+            url = urlparse.urljoin(self.baseurl, link)
+            title = ' "{}"'.format(title) if title.strip() else ""
+            self.o("]({url}{title})".format(url=escape_md(url), title=title))
+
+        if tag == "a" and not self.ignore_links:
+            if start:
+                if (
+                    "href" in attrs
+                    and attrs["href"] is not None
+                    and not (self.skip_internal_links and attrs["href"].startswith("#"))
+                    and not (
+                        self.ignore_mailto_links and attrs["href"].startswith("mailto:")
+                    )
+                ):
+                    self.astack.append(attrs)
+                    self.maybe_automatic_link = attrs["href"]
+                    self.empty_link = True
+                    if self.protect_links:
+                        attrs["href"] = "<" + attrs["href"] + ">"
+                else:
+                    self.astack.append(None)
+            else:
+                if self.astack:
+                    a = self.astack.pop()
+                    if self.maybe_automatic_link and not self.empty_link:
+                        self.maybe_automatic_link = None
+                    elif a:
+                        assert a["href"] is not None
+                        if self.empty_link:
+                            self.o("[")
+                            self.empty_link = False
+                            self.maybe_automatic_link = None
+                        if self.inline_links:
+                            self.p_p = 0
+                            title = a.get("title") or ""
+                            title = escape_md(title)
+                            link_url(self, a["href"], title)
+                        else:
+                            i = self.previousIndex(a)
+                            if i is not None:
+                                a_props = self.a[i]
+                            else:
+                                self.acount += 1
+                                a_props = AnchorElement(a, self.acount, self.outcount)
+                                self.a.append(a_props)
+                            self.o("][" + str(a_props.count) + "]")
+
+        if tag == "img" and start and not self.ignore_images:
+            if "src" in attrs and attrs["src"] is not None:
+                if not self.images_to_alt:
+                    attrs["href"] = attrs["src"]
+                alt = attrs.get("alt") or self.default_image_alt
+
+                # If we have images_with_size, write raw html including width,
+                # height, and alt attributes
+                if self.images_as_html or (
+                    self.images_with_size and ("width" in attrs or "height" in attrs)
+                ):
+                    self.o("<img src='" + attrs["src"] + "' ")
+                    if "width" in attrs and attrs["width"] is not None:
+                        self.o("width='" + attrs["width"] + "' ")
+                    if "height" in attrs and attrs["height"] is not None:
+                        self.o("height='" + attrs["height"] + "' ")
+                    if alt:
+                        self.o("alt='" + alt + "' ")
+                    self.o("/>")
+                    return
+
+                # If we have a link to create, output the start
+                if self.maybe_automatic_link is not None:
+                    href = self.maybe_automatic_link
+                    if (
+                        self.images_to_alt
+                        and escape_md(alt) == href
+                        and self.absolute_url_matcher.match(href)
+                    ):
+                        self.o("<" + escape_md(alt) + ">")
+                        self.empty_link = False
+                        return
+                    else:
+                        self.o("[")
+                        self.maybe_automatic_link = None
+                        self.empty_link = False
+
+                # If we have images_to_alt, we discard the image itself,
+                # considering only the alt text.
+                if self.images_to_alt:
+                    self.o(escape_md(alt))
+                else:
+                    self.o("![" + escape_md(alt) + "]")
+                    if self.inline_links:
+                        href = attrs.get("href") or ""
+                        self.o(
+                            "(" + escape_md(urlparse.urljoin(self.baseurl, href)) + ")"
+                        )
+                    else:
+                        i = self.previousIndex(attrs)
+                        if i is not None:
+                            a_props = self.a[i]
+                        else:
+                            self.acount += 1
+                            a_props = AnchorElement(attrs, self.acount, self.outcount)
+                            self.a.append(a_props)
+                        self.o("[" + str(a_props.count) + "]")
+
+        if tag == "dl" and start:
+            self.p()
+        if tag == "dt" and not start:
+            self.pbr()
+        if tag == "dd" and start:
+            self.o("    ")
+        if tag == "dd" and not start:
+            self.pbr()
+
+        if tag in ["ol", "ul"]:
+            # Google Docs create sub lists as top level lists
+            if not self.list and not self.lastWasList:
+                self.p()
+            if start:
+                if self.google_doc:
+                    list_style = google_list_style(tag_style)
+                else:
+                    list_style = tag
+                numbering_start = list_numbering_start(attrs)
+                self.list.append(ListElement(list_style, numbering_start))
+            else:
+                if self.list:
+                    self.list.pop()
+                    if not self.google_doc and not self.list:
+                        self.o("\n")
+            self.lastWasList = True
+        else:
+            self.lastWasList = False
+
+        if tag == "li":
+            self.pbr()
+            if start:
+                if self.list:
+                    li = self.list[-1]
+                else:
+                    li = ListElement("ul", 0)
+                if self.google_doc:
+                    self.o("  " * self.google_nest_count(tag_style))
+                else:
+                    # Indent two spaces per list, except use three spaces for an
+                    # unordered list inside an ordered list.
+                    # https://spec.commonmark.org/0.28/#motivation
+                    # TODO: line up <ol><li>s > 9 correctly.
+                    parent_list = None
+                    for list in self.list:
+                        self.o(
+                            "   " if parent_list == "ol" and list.name == "ul" else "  "
+                        )
+                        parent_list = list.name
+
+                if li.name == "ul":
+                    self.o(self.ul_item_mark + " ")
+                elif li.name == "ol":
+                    li.num += 1
+                    self.o(str(li.num) + ". ")
+                self.start = True
+
+        if tag in ["table", "tr", "td", "th"]:
+            if self.ignore_tables:
+                if tag == "tr":
+                    if start:
+                        pass
+                    else:
+                        self.soft_br()
+                else:
+                    pass
+
+            elif self.bypass_tables:
+                if start:
+                    self.soft_br()
+                if tag in ["td", "th"]:
+                    if start:
+                        self.o("<{}>\n\n".format(tag))
+                    else:
+                        self.o("\n</{}>".format(tag))
+                else:
+                    if start:
+                        self.o("<{}>".format(tag))
+                    else:
+                        self.o("</{}>".format(tag))
+
+            else:
+                if tag == "table":
+                    if start:
+                        self.table_start = True
+                        if self.pad_tables:
+                            self.o("<" + config.TABLE_MARKER_FOR_PAD + ">")
+                            self.o("  \n")
+                    else:
+                        if self.pad_tables:
+                            # add break in case the table is empty or its 1 row table
+                            self.soft_br()
+                            self.o("</" + config.TABLE_MARKER_FOR_PAD + ">")
+                            self.o("  \n")
+                if tag in ["td", "th"] and start:
+                    if self.split_next_td:
+                        self.o("| ")
+                    self.split_next_td = True
+
+                if tag == "tr" and start:
+                    self.td_count = 0
+                if tag == "tr" and not start:
+                    self.split_next_td = False
+                    self.soft_br()
+                if tag == "tr" and not start and self.table_start:
+                    # Underline table header
+                    self.o("|".join(["---"] * self.td_count))
+                    self.soft_br()
+                    self.table_start = False
+                if tag in ["td", "th"] and start:
+                    self.td_count += 1
+
+        if tag == "pre":
+            if start:
+                self.startpre = True
+                self.pre = True
+            else:
+                self.pre = False
+                if self.mark_code:
+                    self.out("\n[/code]")
+            self.p()
+
+        if tag in ["sup", "sub"] and self.include_sup_sub:
+            if start:
+                self.o("<{}>".format(tag))
+            else:
+                self.o("</{}>".format(tag))
+
+    # TODO: Add docstring for these one letter functions
+    def pbr(self) -> None:
+        "Pretty print has a line break"
+        if self.p_p == 0:
+            self.p_p = 1
+
+    def p(self) -> None:
+        "Set pretty print to 1 or 2 lines"
+        self.p_p = 1 if self.single_line_break else 2
+
+    def soft_br(self) -> None:
+        "Soft breaks"
+        self.pbr()
+        self.br_toggle = "  "
+
+    def o(
+        self, data: str, puredata: bool = False, force: Union[bool, str] = False
+    ) -> None:
+        """
+        Deal with indentation and whitespace
+        """
+        if self.abbr_data is not None:
+            self.abbr_data += data
+
+        if not self.quiet:
+            if self.google_doc:
+                # prevent white space immediately after 'begin emphasis'
+                # marks ('**' and '_')
+                lstripped_data = data.lstrip()
+                if self.drop_white_space and not (self.pre or self.code):
+                    data = lstripped_data
+                if lstripped_data != "":
+                    self.drop_white_space = 0
+
+            if puredata and not self.pre:
+                # This is a very dangerous call ... it could mess up
+                # all handling of &nbsp; when not handled properly
+                # (see entityref)
+                data = re.sub(r"\s+", r" ", data)
+                if data and data[0] == " ":
+                    self.space = True
+                    data = data[1:]
+            if not data and not force:
+                return
+
+            if self.startpre:
+                # self.out(" :") #TODO: not output when already one there
+                if not data.startswith("\n") and not data.startswith("\r\n"):
+                    # <pre>stuff...
+                    data = "\n" + data
+                if self.mark_code:
+                    self.out("\n[code]")
+                    self.p_p = 0
+
+            bq = ">" * self.blockquote
+            if not (force and data and data[0] == ">") and self.blockquote:
+                bq += " "
+
+            if self.pre:
+                if not self.list:
+                    bq += "    "
+                # else: list content is already partially indented
+                bq += "    " * len(self.list)
+                data = data.replace("\n", "\n" + bq)
+
+            if self.startpre:
+                self.startpre = False
+                if self.list:
+                    # use existing initial indentation
+                    data = data.lstrip("\n")
+
+            if self.start:
+                self.space = False
+                self.p_p = 0
+                self.start = False
+
+            if force == "end":
+                # It's the end.
+                self.p_p = 0
+                self.out("\n")
+                self.space = False
+
+            if self.p_p:
+                self.out((self.br_toggle + "\n" + bq) * self.p_p)
+                self.space = False
+                self.br_toggle = ""
+
+            if self.space:
+                if not self.lastWasNL:
+                    self.out(" ")
+                self.space = False
+
+            if self.a and (
+                (self.p_p == 2 and self.links_each_paragraph) or force == "end"
+            ):
+                if force == "end":
+                    self.out("\n")
+
+                newa = []
+                for link in self.a:
+                    if self.outcount > link.outcount:
+                        self.out(
+                            "   ["
+                            + str(link.count)
+                            + "]: "
+                            + urlparse.urljoin(self.baseurl, link.attrs["href"])
+                        )
+                        if "title" in link.attrs and link.attrs["title"] is not None:
+                            self.out(" (" + link.attrs["title"] + ")")
+                        self.out("\n")
+                    else:
+                        newa.append(link)
+
+                # Don't need an extra line when nothing was done.
+                if self.a != newa:
+                    self.out("\n")
+
+                self.a = newa
+
+            if self.abbr_list and force == "end":
+                for abbr, definition in self.abbr_list.items():
+                    self.out("  *[" + abbr + "]: " + definition + "\n")
+
+            self.p_p = 0
+            self.out(data)
+            self.outcount += 1
+
+    def handle_data(self, data: str, entity_char: bool = False) -> None:
+        if not data:
+            # Data may be empty for some HTML entities. For example,
+            # LEFT-TO-RIGHT MARK.
+            return
+
+        if self.stressed:
+            data = data.strip()
+            self.stressed = False
+            self.preceding_stressed = True
+        elif self.preceding_stressed:
+            if (
+                re.match(r"[^][(){}\s.!?]", data[0])
+                and not hn(self.current_tag)
+                and self.current_tag not in ["a", "code", "pre"]
+            ):
+                # should match a letter or common punctuation
+                data = " " + data
+            self.preceding_stressed = False
+
+        if self.style:
+            self.style_def.update(dumb_css_parser(data))
+
+        if self.maybe_automatic_link is not None:
+            href = self.maybe_automatic_link
+            if (
+                href == data
+                and self.absolute_url_matcher.match(href)
+                and self.use_automatic_links
+            ):
+                self.o("<" + data + ">")
+                self.empty_link = False
+                return
+            else:
+                self.o("[")
+                self.maybe_automatic_link = None
+                self.empty_link = False
+
+        if not self.code and not self.pre and not entity_char:
+            data = escape_md_section(data, snob=self.escape_snob, escape_dot=self.escape_dot, escape_plus=self.escape_plus, escape_dash=self.escape_dash)
+        self.preceding_data = data
+        self.o(data, puredata=True)
+
+    def charref(self, name: str) -> str:
+        if name[0] in ["x", "X"]:
+            c = int(name[1:], 16)
+        else:
+            c = int(name)
+
+        if not self.unicode_snob and c in unifiable_n:
+            return unifiable_n[c]
+        else:
+            try:
+                return chr(c)
+            except ValueError:  # invalid unicode
+                return ""
+
+    def entityref(self, c: str) -> str:
+        if not self.unicode_snob and c in config.UNIFIABLE:
+            return config.UNIFIABLE[c]
+        try:
+            ch = html.entities.html5[c + ";"]
+        except KeyError:
+            return "&" + c + ";"
+        return config.UNIFIABLE[c] if c == "nbsp" else ch
+
+    def google_nest_count(self, style: Dict[str, str]) -> int:
+        """
+        Calculate the nesting count of google doc lists
+
+        :type style: dict
+
+        :rtype: int
+        """
+        nest_count = 0
+        if "margin-left" in style:
+            nest_count = int(style["margin-left"][:-2]) // self.google_list_indent
+
+        return nest_count
+
+    def optwrap(self, text: str) -> str:
+        """
+        Wrap all paragraphs in the provided text.
+
+        :type text: str
+
+        :rtype: str
+        """
+        if not self.body_width:
+            return text
+
+        result = ""
+        newlines = 0
+        # I cannot think of a better solution for now.
+        # To avoid the non-wrap behaviour for entire paras
+        # because of the presence of a link in it
+        if not self.wrap_links:
+            self.inline_links = False
+        for para in text.split("\n"):
+            if len(para) > 0:
+                if not skipwrap(
+                    para, self.wrap_links, self.wrap_list_items, self.wrap_tables
+                ):
+                    indent = ""
+                    if para.startswith("  " + self.ul_item_mark):
+                        # list item continuation: add a double indent to the
+                        # new lines
+                        indent = "    "
+                    elif para.startswith("> "):
+                        # blockquote continuation: add the greater than symbol
+                        # to the new lines
+                        indent = "> "
+                    wrapped = wrap(
+                        para,
+                        self.body_width,
+                        break_long_words=False,
+                        subsequent_indent=indent,
+                    )
+                    result += "\n".join(wrapped)
+                    if para.endswith("  "):
+                        result += "  \n"
+                        newlines = 1
+                    elif indent:
+                        result += "\n"
+                        newlines = 1
+                    else:
+                        result += "\n\n"
+                        newlines = 2
+                else:
+                    # Warning for the tempted!!!
+                    # Be aware that obvious replacement of this with
+                    # line.isspace()
+                    # DOES NOT work! Explanations are welcome.
+                    if not config.RE_SPACE.match(para):
+                        result += para + "\n"
+                        newlines = 1
+            else:
+                if newlines < 2:
+                    result += "\n"
+                    newlines += 1
+        return result
+
+def html2text(html: str, baseurl: str = "", bodywidth: Optional[int] = None) -> str:
+    if bodywidth is None:
+        bodywidth = config.BODY_WIDTH
+    h = HTML2Text(baseurl=baseurl, bodywidth=bodywidth)
+
+    return h.handle(html)
+
+class CustomHTML2Text(HTML2Text):
+    def __init__(self, *args, handle_code_in_pre=False, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.inside_pre = False
+        self.inside_code = False
+        self.preserve_tags = set()  # Set of tags to preserve
+        self.current_preserved_tag = None
+        self.preserved_content = []
+        self.preserve_depth = 0
+        self.handle_code_in_pre = handle_code_in_pre 
+        
+        # Configuration options
+        self.skip_internal_links = False
+        self.single_line_break = False
+        self.mark_code = False
+        self.include_sup_sub = False
+        self.body_width = 0
+        self.ignore_mailto_links = True
+        self.ignore_links = False
+        self.escape_backslash = False
+        self.escape_dot = False
+        self.escape_plus = False
+        self.escape_dash = False
+        self.escape_snob = False
+
+    def update_params(self, **kwargs):
+        """Update parameters and set preserved tags."""
+        for key, value in kwargs.items():
+            if key == 'preserve_tags':
+                self.preserve_tags = set(value)
+            elif key == 'handle_code_in_pre':
+                self.handle_code_in_pre = value
+            else:
+                setattr(self, key, value)
+
+    def handle_tag(self, tag, attrs, start):
+        # Handle preserved tags
+        if tag in self.preserve_tags:
+            if start:
+                if self.preserve_depth == 0:
+                    self.current_preserved_tag = tag
+                    self.preserved_content = []
+                    # Format opening tag with attributes
+                    attr_str = ''.join(f' {k}="{v}"' for k, v in attrs.items() if v is not None)
+                    self.preserved_content.append(f'<{tag}{attr_str}>')
+                self.preserve_depth += 1
+                return
+            else:
+                self.preserve_depth -= 1
+                if self.preserve_depth == 0:
+                    self.preserved_content.append(f'</{tag}>')
+                    # Output the preserved HTML block with proper spacing
+                    preserved_html = ''.join(self.preserved_content)
+                    self.o('\n' + preserved_html + '\n')
+                    self.current_preserved_tag = None
+                return
+
+        # If we're inside a preserved tag, collect all content
+        if self.preserve_depth > 0:
+            if start:
+                # Format nested tags with attributes
+                attr_str = ''.join(f' {k}="{v}"' for k, v in attrs.items() if v is not None)
+                self.preserved_content.append(f'<{tag}{attr_str}>')
+            else:
+                self.preserved_content.append(f'</{tag}>')
+            return
+
+        # Handle pre tags
+        if tag == 'pre':
+            if start:
+                self.o('```\n')  # Markdown code block start
+                self.inside_pre = True
+            else:
+                self.o('\n```\n')  # Markdown code block end
+                self.inside_pre = False
+        elif tag == 'code':
+            if self.inside_pre and not self.handle_code_in_pre:
+                # Ignore code tags inside pre blocks if handle_code_in_pre is False
+                return
+            if start:
+                self.o('`')  # Markdown inline code start
+                self.inside_code = True
+            else:
+                self.o('`')  # Markdown inline code end
+                self.inside_code = False
+        else:
+            super().handle_tag(tag, attrs, start)
+
+    def handle_data(self, data, entity_char=False):
+        """Override handle_data to capture content within preserved tags."""
+        if self.preserve_depth > 0:
+            self.preserved_content.append(data)
+            return
+
+        if self.inside_pre:
+            # Output the raw content for pre blocks, including content inside code tags
+            self.o(data)  # Directly output the data as-is (preserve newlines)
+            return
+        if self.inside_code:
+            # Inline code: no newlines allowed
+            self.o(data.replace('\n', ' '))
+            return
+
+        # Default behavior for other tags
+        super().handle_data(data, entity_char)
+
+
+    #     # Handle pre tags
+    #     if tag == 'pre':
+    #         if start:
+    #             self.o('```\n')
+    #             self.inside_pre = True
+    #         else:
+    #             self.o('\n```')
+    #             self.inside_pre = False
+    #     # elif tag in ["h1", "h2", "h3", "h4", "h5", "h6"]:
+    #     #     pass
+    #     else:
+    #         super().handle_tag(tag, attrs, start)
+
+    # def handle_data(self, data, entity_char=False):
+    #     """Override handle_data to capture content within preserved tags."""
+    #     if self.preserve_depth > 0:
+    #         self.preserved_content.append(data)
+    #         return
+    #     super().handle_data(data, entity_char)

crawl4ai/html2text/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+main()

crawl4ai/html2text/_typing.py

@@ -0,0 +1,2 @@
+class OutCallback:
+    def __call__(self, s: str) -> None: ...

crawl4ai/html2text/cli.py

@@ -0,0 +1,330 @@
+import argparse
+import sys
+
+from . import HTML2Text, __version__, config
+
+
+def main() -> None:
+    baseurl = ""
+
+    class bcolors:
+        HEADER = "\033[95m"
+        OKBLUE = "\033[94m"
+        OKGREEN = "\033[92m"
+        WARNING = "\033[93m"
+        FAIL = "\033[91m"
+        ENDC = "\033[0m"
+        BOLD = "\033[1m"
+        UNDERLINE = "\033[4m"
+
+    p = argparse.ArgumentParser()
+    p.add_argument(
+        "--default-image-alt",
+        dest="default_image_alt",
+        default=config.DEFAULT_IMAGE_ALT,
+        help="The default alt string for images with missing ones",
+    )
+    p.add_argument(
+        "--pad-tables",
+        dest="pad_tables",
+        action="store_true",
+        default=config.PAD_TABLES,
+        help="pad the cells to equal column width in tables",
+    )
+    p.add_argument(
+        "--no-wrap-links",
+        dest="wrap_links",
+        action="store_false",
+        default=config.WRAP_LINKS,
+        help="don't wrap links during conversion",
+    )
+    p.add_argument(
+        "--wrap-list-items",
+        dest="wrap_list_items",
+        action="store_true",
+        default=config.WRAP_LIST_ITEMS,
+        help="wrap list items during conversion",
+    )
+    p.add_argument(
+        "--wrap-tables",
+        dest="wrap_tables",
+        action="store_true",
+        default=config.WRAP_TABLES,
+        help="wrap tables",
+    )
+    p.add_argument(
+        "--ignore-emphasis",
+        dest="ignore_emphasis",
+        action="store_true",
+        default=config.IGNORE_EMPHASIS,
+        help="don't include any formatting for emphasis",
+    )
+    p.add_argument(
+        "--reference-links",
+        dest="inline_links",
+        action="store_false",
+        default=config.INLINE_LINKS,
+        help="use reference style links instead of inline links",
+    )
+    p.add_argument(
+        "--ignore-links",
+        dest="ignore_links",
+        action="store_true",
+        default=config.IGNORE_ANCHORS,
+        help="don't include any formatting for links",
+    )
+    p.add_argument(
+        "--ignore-mailto-links",
+        action="store_true",
+        dest="ignore_mailto_links",
+        default=config.IGNORE_MAILTO_LINKS,
+        help="don't include mailto: links",
+    )
+    p.add_argument(
+        "--protect-links",
+        dest="protect_links",
+        action="store_true",
+        default=config.PROTECT_LINKS,
+        help="protect links from line breaks surrounding them with angle brackets",
+    )
+    p.add_argument(
+        "--ignore-images",
+        dest="ignore_images",
+        action="store_true",
+        default=config.IGNORE_IMAGES,
+        help="don't include any formatting for images",
+    )
+    p.add_argument(
+        "--images-as-html",
+        dest="images_as_html",
+        action="store_true",
+        default=config.IMAGES_AS_HTML,
+        help=(
+            "Always write image tags as raw html; preserves `height`, `width` and "
+            "`alt` if possible."
+        ),
+    )
+    p.add_argument(
+        "--images-to-alt",
+        dest="images_to_alt",
+        action="store_true",
+        default=config.IMAGES_TO_ALT,
+        help="Discard image data, only keep alt text",
+    )
+    p.add_argument(
+        "--images-with-size",
+        dest="images_with_size",
+        action="store_true",
+        default=config.IMAGES_WITH_SIZE,
+        help=(
+            "Write image tags with height and width attrs as raw html to retain "
+            "dimensions"
+        ),
+    )
+    p.add_argument(
+        "-g",
+        "--google-doc",
+        action="store_true",
+        dest="google_doc",
+        default=False,
+        help="convert an html-exported Google Document",
+    )
+    p.add_argument(
+        "-d",
+        "--dash-unordered-list",
+        action="store_true",
+        dest="ul_style_dash",
+        default=False,
+        help="use a dash rather than a star for unordered list items",
+    )
+    p.add_argument(
+        "-e",
+        "--asterisk-emphasis",
+        action="store_true",
+        dest="em_style_asterisk",
+        default=False,
+        help="use an asterisk rather than an underscore for emphasized text",
+    )
+    p.add_argument(
+        "-b",
+        "--body-width",
+        dest="body_width",
+        type=int,
+        default=config.BODY_WIDTH,
+        help="number of characters per output line, 0 for no wrap",
+    )
+    p.add_argument(
+        "-i",
+        "--google-list-indent",
+        dest="list_indent",
+        type=int,
+        default=config.GOOGLE_LIST_INDENT,
+        help="number of pixels Google indents nested lists",
+    )
+    p.add_argument(
+        "-s",
+        "--hide-strikethrough",
+        action="store_true",
+        dest="hide_strikethrough",
+        default=False,
+        help="hide strike-through text. only relevant when -g is " "specified as well",
+    )
+    p.add_argument(
+        "--escape-all",
+        action="store_true",
+        dest="escape_snob",
+        default=False,
+        help=(
+            "Escape all special characters.  Output is less readable, but avoids "
+            "corner case formatting issues."
+        ),
+    )
+    p.add_argument(
+        "--bypass-tables",
+        action="store_true",
+        dest="bypass_tables",
+        default=config.BYPASS_TABLES,
+        help="Format tables in HTML rather than Markdown syntax.",
+    )
+    p.add_argument(
+        "--ignore-tables",
+        action="store_true",
+        dest="ignore_tables",
+        default=config.IGNORE_TABLES,
+        help="Ignore table-related tags (table, th, td, tr) " "while keeping rows.",
+    )
+    p.add_argument(
+        "--single-line-break",
+        action="store_true",
+        dest="single_line_break",
+        default=config.SINGLE_LINE_BREAK,
+        help=(
+            "Use a single line break after a block element rather than two line "
+            "breaks. NOTE: Requires --body-width=0"
+        ),
+    )
+    p.add_argument(
+        "--unicode-snob",
+        action="store_true",
+        dest="unicode_snob",
+        default=config.UNICODE_SNOB,
+        help="Use unicode throughout document",
+    )
+    p.add_argument(
+        "--no-automatic-links",
+        action="store_false",
+        dest="use_automatic_links",
+        default=config.USE_AUTOMATIC_LINKS,
+        help="Do not use automatic links wherever applicable",
+    )
+    p.add_argument(
+        "--no-skip-internal-links",
+        action="store_false",
+        dest="skip_internal_links",
+        default=config.SKIP_INTERNAL_LINKS,
+        help="Do not skip internal links",
+    )
+    p.add_argument(
+        "--links-after-para",
+        action="store_true",
+        dest="links_each_paragraph",
+        default=config.LINKS_EACH_PARAGRAPH,
+        help="Put links after each paragraph instead of document",
+    )
+    p.add_argument(
+        "--mark-code",
+        action="store_true",
+        dest="mark_code",
+        default=config.MARK_CODE,
+        help="Mark program code blocks with [code]...[/code]",
+    )
+    p.add_argument(
+        "--decode-errors",
+        dest="decode_errors",
+        default=config.DECODE_ERRORS,
+        help=(
+            "What to do in case of decode errors.'ignore', 'strict' and 'replace' are "
+            "acceptable values"
+        ),
+    )
+    p.add_argument(
+        "--open-quote",
+        dest="open_quote",
+        default=config.OPEN_QUOTE,
+        help="The character used to open quotes",
+    )
+    p.add_argument(
+        "--close-quote",
+        dest="close_quote",
+        default=config.CLOSE_QUOTE,
+        help="The character used to close quotes",
+    )
+    p.add_argument(
+        "--version", action="version", version=".".join(map(str, __version__))
+    )
+    p.add_argument("filename", nargs="?")
+    p.add_argument("encoding", nargs="?", default="utf-8")
+    p.add_argument(
+        "--include-sup-sub",
+        dest="include_sup_sub",
+        action="store_true",
+        default=config.INCLUDE_SUP_SUB,
+        help="Include the sup and sub tags",
+    )
+    args = p.parse_args()
+
+    if args.filename and args.filename != "-":
+        with open(args.filename, "rb") as fp:
+            data = fp.read()
+    else:
+        data = sys.stdin.buffer.read()
+
+    try:
+        html = data.decode(args.encoding, args.decode_errors)
+    except UnicodeDecodeError as err:
+        warning = bcolors.WARNING + "Warning:" + bcolors.ENDC
+        warning += " Use the " + bcolors.OKGREEN
+        warning += "--decode-errors=ignore" + bcolors.ENDC + " flag."
+        print(warning)
+        raise err
+
+    h = HTML2Text(baseurl=baseurl)
+    # handle options
+    if args.ul_style_dash:
+        h.ul_item_mark = "-"
+    if args.em_style_asterisk:
+        h.emphasis_mark = "*"
+        h.strong_mark = "__"
+
+    h.body_width = args.body_width
+    h.google_list_indent = args.list_indent
+    h.ignore_emphasis = args.ignore_emphasis
+    h.ignore_links = args.ignore_links
+    h.ignore_mailto_links = args.ignore_mailto_links
+    h.protect_links = args.protect_links
+    h.ignore_images = args.ignore_images
+    h.images_as_html = args.images_as_html
+    h.images_to_alt = args.images_to_alt
+    h.images_with_size = args.images_with_size
+    h.google_doc = args.google_doc
+    h.hide_strikethrough = args.hide_strikethrough
+    h.escape_snob = args.escape_snob
+    h.bypass_tables = args.bypass_tables
+    h.ignore_tables = args.ignore_tables
+    h.single_line_break = args.single_line_break
+    h.inline_links = args.inline_links
+    h.unicode_snob = args.unicode_snob
+    h.use_automatic_links = args.use_automatic_links
+    h.skip_internal_links = args.skip_internal_links
+    h.links_each_paragraph = args.links_each_paragraph
+    h.mark_code = args.mark_code
+    h.wrap_links = args.wrap_links
+    h.wrap_list_items = args.wrap_list_items
+    h.wrap_tables = args.wrap_tables
+    h.pad_tables = args.pad_tables
+    h.default_image_alt = args.default_image_alt
+    h.open_quote = args.open_quote
+    h.close_quote = args.close_quote
+    h.include_sup_sub = args.include_sup_sub
+
+    sys.stdout.write(h.handle(html))

crawl4ai/html2text/config.py

@@ -0,0 +1,172 @@
+import re
+
+# Use Unicode characters instead of their ascii pseudo-replacements
+UNICODE_SNOB = False
+
+# Marker to use for marking tables for padding post processing
+TABLE_MARKER_FOR_PAD = "special_marker_for_table_padding"
+# Escape all special characters.  Output is less readable, but avoids
+# corner case formatting issues.
+ESCAPE_SNOB = False
+ESCAPE_BACKSLASH = False
+ESCAPE_DOT = False
+ESCAPE_PLUS = False
+ESCAPE_DASH = False
+
+# Put the links after each paragraph instead of at the end.
+LINKS_EACH_PARAGRAPH = False
+
+# Wrap long lines at position. 0 for no wrapping.
+BODY_WIDTH = 78
+
+# Don't show internal links (href="#local-anchor") -- corresponding link
+# targets won't be visible in the plain text file anyway.
+SKIP_INTERNAL_LINKS = True
+
+# Use inline, rather than reference, formatting for images and links
+INLINE_LINKS = True
+
+# Protect links from line breaks surrounding them with angle brackets (in
+# addition to their square brackets)
+PROTECT_LINKS = False
+# WRAP_LINKS = True
+WRAP_LINKS = True
+
+# Wrap list items.
+WRAP_LIST_ITEMS = False
+
+# Wrap tables
+WRAP_TABLES = False
+
+# Number of pixels Google indents nested lists
+GOOGLE_LIST_INDENT = 36
+
+# Values Google and others may use to indicate bold text
+BOLD_TEXT_STYLE_VALUES = ("bold", "700", "800", "900")
+
+IGNORE_ANCHORS = False
+IGNORE_MAILTO_LINKS = False
+IGNORE_IMAGES = False
+IMAGES_AS_HTML = False
+IMAGES_TO_ALT = False
+IMAGES_WITH_SIZE = False
+IGNORE_EMPHASIS = False
+MARK_CODE = False
+DECODE_ERRORS = "strict"
+DEFAULT_IMAGE_ALT = ""
+PAD_TABLES = False
+
+# Convert links with same href and text to <href> format
+# if they are absolute links
+USE_AUTOMATIC_LINKS = True
+
+# For checking space-only lines on line 771
+RE_SPACE = re.compile(r"\s\+")
+
+RE_ORDERED_LIST_MATCHER = re.compile(r"\d+\.\s")
+RE_UNORDERED_LIST_MATCHER = re.compile(r"[-\*\+]\s")
+RE_MD_CHARS_MATCHER = re.compile(r"([\\\[\]\(\)])")
+RE_MD_CHARS_MATCHER_ALL = re.compile(r"([`\*_{}\[\]\(\)#!])")
+
+# to find links in the text
+RE_LINK = re.compile(r"(\[.*?\] ?\(.*?\))|(\[.*?\]:.*?)")
+
+# to find table separators
+RE_TABLE = re.compile(r" \| ")
+
+RE_MD_DOT_MATCHER = re.compile(
+    r"""
+    ^             # start of line
+    (\s*\d+)      # optional whitespace and a number
+    (\.)          # dot
+    (?=\s)        # lookahead assert whitespace
+    """,
+    re.MULTILINE | re.VERBOSE,
+)
+RE_MD_PLUS_MATCHER = re.compile(
+    r"""
+    ^
+    (\s*)
+    (\+)
+    (?=\s)
+    """,
+    flags=re.MULTILINE | re.VERBOSE,
+)
+RE_MD_DASH_MATCHER = re.compile(
+    r"""
+    ^
+    (\s*)
+    (-)
+    (?=\s|\-)     # followed by whitespace (bullet list, or spaced out hr)
+                  # or another dash (header or hr)
+    """,
+    flags=re.MULTILINE | re.VERBOSE,
+)
+RE_SLASH_CHARS = r"\`*_{}[]()#+-.!"
+RE_MD_BACKSLASH_MATCHER = re.compile(
+    r"""
+    (\\)          # match one slash
+    (?=[%s])      # followed by a char that requires escaping
+    """
+    % re.escape(RE_SLASH_CHARS),
+    flags=re.VERBOSE,
+)
+
+UNIFIABLE = {
+    "rsquo": "'",
+    "lsquo": "'",
+    "rdquo": '"',
+    "ldquo": '"',
+    "copy": "(C)",
+    "mdash": "--",
+    "nbsp": " ",
+    "rarr": "->",
+    "larr": "<-",
+    "middot": "*",
+    "ndash": "-",
+    "oelig": "oe",
+    "aelig": "ae",
+    "agrave": "a",
+    "aacute": "a",
+    "acirc": "a",
+    "atilde": "a",
+    "auml": "a",
+    "aring": "a",
+    "egrave": "e",
+    "eacute": "e",
+    "ecirc": "e",
+    "euml": "e",
+    "igrave": "i",
+    "iacute": "i",
+    "icirc": "i",
+    "iuml": "i",
+    "ograve": "o",
+    "oacute": "o",
+    "ocirc": "o",
+    "otilde": "o",
+    "ouml": "o",
+    "ugrave": "u",
+    "uacute": "u",
+    "ucirc": "u",
+    "uuml": "u",
+    "lrm": "",
+    "rlm": "",
+}
+
+# Format tables in HTML rather than Markdown syntax
+BYPASS_TABLES = False
+# Ignore table-related tags (table, th, td, tr) while keeping rows
+IGNORE_TABLES = False
+
+
+# Use a single line break after a block element rather than two line breaks.
+# NOTE: Requires body width setting to be 0.
+SINGLE_LINE_BREAK = False
+
+
+# Use double quotation marks when converting the <q> tag.
+OPEN_QUOTE = '"'
+CLOSE_QUOTE = '"'
+
+# Include the <sup> and <sub> tags
+INCLUDE_SUP_SUB = False

crawl4ai/html2text/elements.py

@@ -0,0 +1,18 @@
+from typing import Dict, Optional
+
+
+class AnchorElement:
+    __slots__ = ["attrs", "count", "outcount"]
+
+    def __init__(self, attrs: Dict[str, Optional[str]], count: int, outcount: int):
+        self.attrs = attrs
+        self.count = count
+        self.outcount = outcount
+
+
+class ListElement:
+    __slots__ = ["name", "num"]
+
+    def __init__(self, name: str, num: int):
+        self.name = name
+        self.num = num

crawl4ai/html2text/utils.py

@@ -0,0 +1,303 @@
+import html.entities
+from typing import Dict, List, Optional
+
+from . import config
+
+unifiable_n = {
+    html.entities.name2codepoint[k]: v
+    for k, v in config.UNIFIABLE.items()
+    if k != "nbsp"
+}
+
+
+def hn(tag: str) -> int:
+    if tag[0] == "h" and len(tag) == 2:
+        n = tag[1]
+        if "0" < n <= "9":
+            return int(n)
+    return 0
+
+
+def dumb_property_dict(style: str) -> Dict[str, str]:
+    """
+    :returns: A hash of css attributes
+    """
+    return {
+        x.strip().lower(): y.strip().lower()
+        for x, y in [z.split(":", 1) for z in style.split(";") if ":" in z]
+    }
+
+
+def dumb_css_parser(data: str) -> Dict[str, Dict[str, str]]:
+    """
+    :type data: str
+
+    :returns: A hash of css selectors, each of which contains a hash of
+    css attributes.
+    :rtype: dict
+    """
+    # remove @import sentences
+    data += ";"
+    importIndex = data.find("@import")
+    while importIndex != -1:
+        data = data[0:importIndex] + data[data.find(";", importIndex) + 1 :]
+        importIndex = data.find("@import")
+
+    # parse the css. reverted from dictionary comprehension in order to
+    # support older pythons
+    pairs = [x.split("{") for x in data.split("}") if "{" in x.strip()]
+    try:
+        elements = {a.strip(): dumb_property_dict(b) for a, b in pairs}
+    except ValueError:
+        elements = {}  # not that important
+
+    return elements
+
+
+def element_style(
+    attrs: Dict[str, Optional[str]],
+    style_def: Dict[str, Dict[str, str]],
+    parent_style: Dict[str, str],
+) -> Dict[str, str]:
+    """
+    :type attrs: dict
+    :type style_def: dict
+    :type style_def: dict
+
+    :returns: A hash of the 'final' style attributes of the element
+    :rtype: dict
+    """
+    style = parent_style.copy()
+    if "class" in attrs:
+        assert attrs["class"] is not None
+        for css_class in attrs["class"].split():
+            css_style = style_def.get("." + css_class, {})
+            style.update(css_style)
+    if "style" in attrs:
+        assert attrs["style"] is not None
+        immediate_style = dumb_property_dict(attrs["style"])
+        style.update(immediate_style)
+
+    return style
+
+
+def google_list_style(style: Dict[str, str]) -> str:
+    """
+    Finds out whether this is an ordered or unordered list
+
+    :type style: dict
+
+    :rtype: str
+    """
+    if "list-style-type" in style:
+        list_style = style["list-style-type"]
+        if list_style in ["disc", "circle", "square", "none"]:
+            return "ul"
+
+    return "ol"
+
+
+def google_has_height(style: Dict[str, str]) -> bool:
+    """
+    Check if the style of the element has the 'height' attribute
+    explicitly defined
+
+    :type style: dict
+
+    :rtype: bool
+    """
+    return "height" in style
+
+
+def google_text_emphasis(style: Dict[str, str]) -> List[str]:
+    """
+    :type style: dict
+
+    :returns: A list of all emphasis modifiers of the element
+    :rtype: list
+    """
+    emphasis = []
+    if "text-decoration" in style:
+        emphasis.append(style["text-decoration"])
+    if "font-style" in style:
+        emphasis.append(style["font-style"])
+    if "font-weight" in style:
+        emphasis.append(style["font-weight"])
+
+    return emphasis
+
+
+def google_fixed_width_font(style: Dict[str, str]) -> bool:
+    """
+    Check if the css of the current element defines a fixed width font
+
+    :type style: dict
+
+    :rtype: bool
+    """
+    font_family = ""
+    if "font-family" in style:
+        font_family = style["font-family"]
+    return "courier new" == font_family or "consolas" == font_family
+
+
+def list_numbering_start(attrs: Dict[str, Optional[str]]) -> int:
+    """
+    Extract numbering from list element attributes
+
+    :type attrs: dict
+
+    :rtype: int or None
+    """
+    if "start" in attrs:
+        assert attrs["start"] is not None
+        try:
+            return int(attrs["start"]) - 1
+        except ValueError:
+            pass
+
+    return 0
+
+
+def skipwrap(
+    para: str, wrap_links: bool, wrap_list_items: bool, wrap_tables: bool
+) -> bool:
+    # If it appears to contain a link
+    # don't wrap
+    if not wrap_links and config.RE_LINK.search(para):
+        return True
+    # If the text begins with four spaces or one tab, it's a code block;
+    # don't wrap
+    if para[0:4] == "    " or para[0] == "\t":
+        return True
+
+    # If the text begins with only two "--", possibly preceded by
+    # whitespace, that's an emdash; so wrap.
+    stripped = para.lstrip()
+    if stripped[0:2] == "--" and len(stripped) > 2 and stripped[2] != "-":
+        return False
+
+    # I'm not sure what this is for; I thought it was to detect lists,
+    # but there's a <br>-inside-<span> case in one of the tests that
+    # also depends upon it.
+    if stripped[0:1] in ("-", "*") and not stripped[0:2] == "**":
+        return not wrap_list_items
+
+    # If text contains a pipe character it is likely a table
+    if not wrap_tables and config.RE_TABLE.search(para):
+        return True
+
+    # If the text begins with a single -, *, or +, followed by a space,
+    # or an integer, followed by a ., followed by a space (in either
+    # case optionally proceeded by whitespace), it's a list; don't wrap.
+    return bool(
+        config.RE_ORDERED_LIST_MATCHER.match(stripped)
+        or config.RE_UNORDERED_LIST_MATCHER.match(stripped)
+    )
+
+
+def escape_md(text: str) -> str:
+    """
+    Escapes markdown-sensitive characters within other markdown
+    constructs.
+    """
+    return config.RE_MD_CHARS_MATCHER.sub(r"\\\1", text)
+
+
+def escape_md_section(
+    text: str,
+    escape_backslash: bool = True,
+    snob: bool = False,
+    escape_dot: bool = True,
+    escape_plus: bool = True,
+    escape_dash: bool = True
+) -> str:
+    """
+    Escapes markdown-sensitive characters across whole document sections.
+    Each escaping operation can be controlled individually.
+    """
+    if escape_backslash:
+        text = config.RE_MD_BACKSLASH_MATCHER.sub(r"\\\1", text)
+
+    if snob:
+        text = config.RE_MD_CHARS_MATCHER_ALL.sub(r"\\\1", text)
+
+    if escape_dot:
+        text = config.RE_MD_DOT_MATCHER.sub(r"\1\\\2", text)
+
+    if escape_plus:
+        text = config.RE_MD_PLUS_MATCHER.sub(r"\1\\\2", text)
+
+    if escape_dash:
+        text = config.RE_MD_DASH_MATCHER.sub(r"\1\\\2", text)
+
+    return text
+
+def reformat_table(lines: List[str], right_margin: int) -> List[str]:
+    """
+    Given the lines of a table
+    padds the cells and returns the new lines
+    """
+    # find the maximum width of the columns
+    max_width = [len(x.rstrip()) + right_margin for x in lines[0].split("|")]
+    max_cols = len(max_width)
+    for line in lines:
+        cols = [x.rstrip() for x in line.split("|")]
+        num_cols = len(cols)
+
+        # don't drop any data if colspan attributes result in unequal lengths
+        if num_cols < max_cols:
+            cols += [""] * (max_cols - num_cols)
+        elif max_cols < num_cols:
+            max_width += [len(x) + right_margin for x in cols[-(num_cols - max_cols) :]]
+            max_cols = num_cols
+
+        max_width = [
+            max(len(x) + right_margin, old_len) for x, old_len in zip(cols, max_width)
+        ]
+
+    # reformat
+    new_lines = []
+    for line in lines:
+        cols = [x.rstrip() for x in line.split("|")]
+        if set(line.strip()) == set("-|"):
+            filler = "-"
+            new_cols = [
+                x.rstrip() + (filler * (M - len(x.rstrip())))
+                for x, M in zip(cols, max_width)
+            ]
+            new_lines.append("|-" + "|".join(new_cols) + "|")
+        else:
+            filler = " "
+            new_cols = [
+                x.rstrip() + (filler * (M - len(x.rstrip())))
+                for x, M in zip(cols, max_width)
+            ]
+            new_lines.append("| " + "|".join(new_cols) + "|")
+    return new_lines
+
+
+def pad_tables_in_text(text: str, right_margin: int = 1) -> str:
+    """
+    Provide padding for tables in the text
+    """
+    lines = text.split("\n")
+    table_buffer = []  # type: List[str]
+    table_started = False
+    new_lines = []
+    for line in lines:
+        # Toggle table started
+        if config.TABLE_MARKER_FOR_PAD in line:
+            table_started = not table_started
+            if not table_started:
+                table = reformat_table(table_buffer, right_margin)
+                new_lines.extend(table)
+                table_buffer = []
+                new_lines.append("")
+            continue
+        # Process lines
+        if table_started:
+            table_buffer.append(line)
+        else:
+            new_lines.append(line)
+    return "\n".join(new_lines)

crawl4ai/install.py

@@ -0,0 +1,83 @@
+import subprocess
+import sys
+import asyncio
+from .async_logger import AsyncLogger, LogLevel
+
+# Initialize logger
+logger = AsyncLogger(log_level=LogLevel.DEBUG, verbose=True)
+
+def post_install():
+    """Run all post-installation tasks"""
+    logger.info("Running post-installation setup...", tag="INIT")
+    install_playwright()
+    run_migration()
+    logger.success("Post-installation setup completed!", tag="COMPLETE")
+    
+def install_playwright():
+    logger.info("Installing Playwright browsers...", tag="INIT")
+    try:
+        # subprocess.check_call([sys.executable, "-m", "playwright", "install", "--with-deps", "--force", "chrome"])
+        subprocess.check_call([sys.executable, "-m", "playwright", "install", "--with-deps", "--force", "chromium"])
+        logger.success("Playwright installation completed successfully.", tag="COMPLETE")
+    except subprocess.CalledProcessError as e:
+        # logger.error(f"Error during Playwright installation: {e}", tag="ERROR")
+        logger.warning(f"Please run '{sys.executable} -m playwright install --with-deps' manually after the installation.")
+    except Exception as e:
+        # logger.error(f"Unexpected error during Playwright installation: {e}", tag="ERROR")
+        logger.warning(f"Please run '{sys.executable} -m playwright install --with-deps' manually after the installation.")
+
+def run_migration():
+    """Initialize database during installation"""
+    try:
+        logger.info("Starting database initialization...", tag="INIT")
+        from crawl4ai.async_database import async_db_manager
+
+        asyncio.run(async_db_manager.initialize())
+        logger.success("Database initialization completed successfully.", tag="COMPLETE")
+    except ImportError:
+        logger.warning("Database module not found. Will initialize on first use.")
+    except Exception as e:
+        logger.warning(f"Database initialization failed: {e}")
+        logger.warning("Database will be initialized on first use")
+
+async def run_doctor():
+    """Test if Crawl4AI is working properly"""
+    logger.info("Running Crawl4AI health check...", tag="INIT")
+    try:
+        from .async_webcrawler import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+
+        browser_config = BrowserConfig(
+            headless=True,
+            browser_type="chromium",
+            ignore_https_errors=True,
+            light_mode=True,
+            viewport_width=1280,
+            viewport_height=720
+        )
+
+        run_config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            screenshot=True,
+        )
+
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            logger.info("Testing crawling capabilities...", tag="TEST")
+            result = await crawler.arun(
+                url="https://crawl4ai.com",
+                config=run_config
+            )
+
+            if result and result.markdown:
+                logger.success("✅ Crawling test passed!", tag="COMPLETE")
+                return True
+            else:
+                raise Exception("Failed to get content")
+
+    except Exception as e:
+        logger.error(f"❌ Test failed: {e}", tag="ERROR")
+        return False
+
+def doctor():
+    """Entry point for the doctor command"""
+    import asyncio
+    return asyncio.run(run_doctor())

crawl4ai/js_snippet/__init__.py

@@ -0,0 +1,15 @@
+import os, sys
+
+# Create a function get name of a js script, then load from the CURRENT folder of this script and return its content as string, make sure its error free
+def load_js_script(script_name):
+    # Get the path of the current script
+    current_script_path = os.path.dirname(os.path.realpath(__file__))
+    # Get the path of the script to load
+    script_path = os.path.join(current_script_path, script_name + '.js')
+    # Check if the script exists
+    if not os.path.exists(script_path):
+        raise ValueError(f"Script {script_name} not found in the folder {current_script_path}")
+    # Load the content of the script
+    with open(script_path, 'r') as f:
+        script_content = f.read()
+    return script_content

crawl4ai/js_snippet/navigator_overrider.js

@@ -0,0 +1,25 @@
+// Pass the Permissions Test.
+const originalQuery = window.navigator.permissions.query;
+window.navigator.permissions.query = (parameters) =>
+    parameters.name === "notifications"
+        ? Promise.resolve({ state: Notification.permission })
+        : originalQuery(parameters);
+Object.defineProperty(navigator, "webdriver", {
+    get: () => undefined,
+});
+window.navigator.chrome = {
+    runtime: {},
+    // Add other properties if necessary
+};
+Object.defineProperty(navigator, "plugins", {
+    get: () => [1, 2, 3, 4, 5],
+});
+Object.defineProperty(navigator, "languages", {
+    get: () => ["en-US", "en"],
+});
+Object.defineProperty(document, "hidden", {
+    get: () => false,
+});
+Object.defineProperty(document, "visibilityState", {
+    get: () => "visible",
+});

crawl4ai/js_snippet/remove_overlay_elements.js

@@ -0,0 +1,119 @@
+async () => {
+    // Function to check if element is visible
+    const isVisible = (elem) => {
+        const style = window.getComputedStyle(elem);
+        return style.display !== "none" && style.visibility !== "hidden" && style.opacity !== "0";
+    };
+
+    // Common selectors for popups and overlays
+    const commonSelectors = [
+        // Close buttons first
+        'button[class*="close" i]',
+        'button[class*="dismiss" i]',
+        'button[aria-label*="close" i]',
+        'button[title*="close" i]',
+        'a[class*="close" i]',
+        'span[class*="close" i]',
+
+        // Cookie notices
+        '[class*="cookie-banner" i]',
+        '[id*="cookie-banner" i]',
+        '[class*="cookie-consent" i]',
+        '[id*="cookie-consent" i]',
+
+        // Newsletter/subscription dialogs
+        '[class*="newsletter" i]',
+        '[class*="subscribe" i]',
+
+        // Generic popups/modals
+        '[class*="popup" i]',
+        '[class*="modal" i]',
+        '[class*="overlay" i]',
+        '[class*="dialog" i]',
+        '[role="dialog"]',
+        '[role="alertdialog"]',
+    ];
+
+    // Try to click close buttons first
+    for (const selector of commonSelectors.slice(0, 6)) {
+        const closeButtons = document.querySelectorAll(selector);
+        for (const button of closeButtons) {
+            if (isVisible(button)) {
+                try {
+                    button.click();
+                    await new Promise((resolve) => setTimeout(resolve, 100));
+                } catch (e) {
+                    console.log("Error clicking button:", e);
+                }
+            }
+        }
+    }
+
+    // Remove remaining overlay elements
+    const removeOverlays = () => {
+        // Find elements with high z-index
+        const allElements = document.querySelectorAll("*");
+        for (const elem of allElements) {
+            const style = window.getComputedStyle(elem);
+            const zIndex = parseInt(style.zIndex);
+            const position = style.position;
+
+            if (
+                isVisible(elem) &&
+                (zIndex > 999 || position === "fixed" || position === "absolute") &&
+                (elem.offsetWidth > window.innerWidth * 0.5 ||
+                    elem.offsetHeight > window.innerHeight * 0.5 ||
+                    style.backgroundColor.includes("rgba") ||
+                    parseFloat(style.opacity) < 1)
+            ) {
+                elem.remove();
+            }
+        }
+
+        // Remove elements matching common selectors
+        for (const selector of commonSelectors) {
+            const elements = document.querySelectorAll(selector);
+            elements.forEach((elem) => {
+                if (isVisible(elem)) {
+                    elem.remove();
+                }
+            });
+        }
+    };
+
+    // Remove overlay elements
+    removeOverlays();
+
+    // Remove any fixed/sticky position elements at the top/bottom
+    const removeFixedElements = () => {
+        const elements = document.querySelectorAll("*");
+        elements.forEach((elem) => {
+            const style = window.getComputedStyle(elem);
+            if ((style.position === "fixed" || style.position === "sticky") && isVisible(elem)) {
+                elem.remove();
+            }
+        });
+    };
+
+    removeFixedElements();
+
+    // Remove empty block elements as: div, p, span, etc.
+    const removeEmptyBlockElements = () => {
+        const blockElements = document.querySelectorAll(
+            "div, p, span, section, article, header, footer, aside, nav, main, ul, ol, li, dl, dt, dd, h1, h2, h3, h4, h5, h6"
+        );
+        blockElements.forEach((elem) => {
+            if (elem.innerText.trim() === "") {
+                elem.remove();
+            }
+        });
+    };
+
+    // Remove margin-right and padding-right from body (often added by modal scripts)
+    document.body.style.marginRight = "0px";
+    document.body.style.paddingRight = "0px";
+    document.body.style.overflow = "auto";
+
+    // Wait a bit for any animations to complete
+    await new Promise((resolve) => setTimeout(resolve, 100));
+};

crawl4ai/js_snippet/update_image_dimensions.js

@@ -0,0 +1,54 @@
+() => {
+    return new Promise((resolve) => {
+        const filterImage = (img) => {
+            // Filter out images that are too small
+            if (img.width < 100 && img.height < 100) return false;
+
+            // Filter out images that are not visible
+            const rect = img.getBoundingClientRect();
+            if (rect.width === 0 || rect.height === 0) return false;
+
+            // Filter out images with certain class names (e.g., icons, thumbnails)
+            if (img.classList.contains("icon") || img.classList.contains("thumbnail")) return false;
+
+            // Filter out images with certain patterns in their src (e.g., placeholder images)
+            if (img.src.includes("placeholder") || img.src.includes("icon")) return false;
+
+            return true;
+        };
+
+        const images = Array.from(document.querySelectorAll("img")).filter(filterImage);
+        let imagesLeft = images.length;
+
+        if (imagesLeft === 0) {
+            resolve();
+            return;
+        }
+
+        const checkImage = (img) => {
+            if (img.complete && img.naturalWidth !== 0) {
+                img.setAttribute("width", img.naturalWidth);
+                img.setAttribute("height", img.naturalHeight);
+                imagesLeft--;
+                if (imagesLeft === 0) resolve();
+            }
+        };
+
+        images.forEach((img) => {
+            checkImage(img);
+            if (!img.complete) {
+                img.onload = () => {
+                    checkImage(img);
+                };
+                img.onerror = () => {
+                    imagesLeft--;
+                    if (imagesLeft === 0) resolve();
+                };
+            }
+        });
+
+        // Fallback timeout of 5 seconds
+        // setTimeout(() => resolve(), 5000);
+        resolve();
+    });
+};

crawl4ai/llmtxt.py

@@ -0,0 +1,498 @@
+import os
+from pathlib import Path
+import re
+from typing import Dict, List, Tuple, Optional, Any
+import json
+from tqdm import tqdm
+import time
+import psutil
+import numpy as np
+from rank_bm25 import BM25Okapi
+from nltk.tokenize import word_tokenize
+from nltk.corpus import stopwords
+from nltk.stem import WordNetLemmatizer
+from litellm import completion, batch_completion
+from .async_logger import AsyncLogger
+import litellm
+import pickle
+import hashlib  # <--- ADDED for file-hash
+from fnmatch import fnmatch
+import glob
+
+litellm.set_verbose = False
+
+def _compute_file_hash(file_path: Path) -> str:
+    """Compute MD5 hash for the file's entire content."""
+    hash_md5 = hashlib.md5()
+    with file_path.open("rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            hash_md5.update(chunk)
+    return hash_md5.hexdigest()
+
+class AsyncLLMTextManager:
+    def __init__(
+        self,
+        docs_dir: Path,
+        logger: Optional[AsyncLogger] = None,
+        max_concurrent_calls: int = 5,
+        batch_size: int = 3
+    ) -> None:
+        self.docs_dir = docs_dir
+        self.logger = logger
+        self.max_concurrent_calls = max_concurrent_calls
+        self.batch_size = batch_size
+        self.bm25_index = None
+        self.document_map: Dict[str, Any] = {}
+        self.tokenized_facts: List[str] = []
+        self.bm25_index_file = self.docs_dir / "bm25_index.pkl"
+
+    async def _process_document_batch(self, doc_batch: List[Path]) -> None:
+        """Process a batch of documents in parallel"""
+        contents = []
+        for file_path in doc_batch:
+            try:
+                with open(file_path, 'r', encoding='utf-8') as f:
+                    contents.append(f.read())
+            except Exception as e:
+                self.logger.error(f"Error reading {file_path}: {str(e)}")
+                contents.append("")  # Add empty content to maintain batch alignment
+
+        prompt = """Given a documentation file, generate a list of atomic facts where each fact:
+1. Represents a single piece of knowledge
+2. Contains variations in terminology for the same concept
+3. References relevant code patterns if they exist
+4. Is written in a way that would match natural language queries
+
+Each fact should follow this format:
+<main_concept>: <fact_statement> | <related_terms> | <code_reference>
+
+Example Facts:
+browser_config: Configure headless mode and browser type for AsyncWebCrawler | headless, browser_type, chromium, firefox | BrowserConfig(browser_type="chromium", headless=True)
+redis_connection: Redis client connection requires host and port configuration | redis setup, redis client, connection params | Redis(host='localhost', port=6379, db=0)
+pandas_filtering: Filter DataFrame rows using boolean conditions | dataframe filter, query, boolean indexing | df[df['column'] > 5]
+
+Wrap your response in <index>...</index> tags.
+"""
+
+        # Prepare messages for batch processing
+        messages_list = [
+            [
+                {"role": "user", "content": f"{prompt}\n\nGenerate index for this documentation:\n\n{content}"}
+            ]
+            for content in contents if content
+        ]
+
+        try:
+            responses = batch_completion(
+                model="anthropic/claude-3-5-sonnet-latest",
+                messages=messages_list,
+                logger_fn=None
+            )
+
+            # Process responses and save index files
+            for response, file_path in zip(responses, doc_batch):
+                try:
+                    index_content_match = re.search(
+                        r'<index>(.*?)</index>',
+                        response.choices[0].message.content,
+                        re.DOTALL
+                    )
+                    if not index_content_match:
+                        self.logger.warning(f"No <index>...</index> content found for {file_path}")
+                        continue
+
+                    index_content = re.sub(
+                        r"\n\s*\n", "\n", index_content_match.group(1)
+                    ).strip()
+                    if index_content:
+                        index_file = file_path.with_suffix('.q.md')
+                        with open(index_file, 'w', encoding='utf-8') as f:
+                            f.write(index_content)
+                        self.logger.info(f"Created index file: {index_file}")
+                    else:
+                        self.logger.warning(f"No index content found in response for {file_path}")
+
+                except Exception as e:
+                    self.logger.error(f"Error processing response for {file_path}: {str(e)}")
+
+        except Exception as e:
+            self.logger.error(f"Error in batch completion: {str(e)}")
+
+    def _validate_fact_line(self, line: str) -> Tuple[bool, Optional[str]]:
+        if "|" not in line:
+            return False, "Missing separator '|'"
+
+        parts = [p.strip() for p in line.split("|")]
+        if len(parts) != 3:
+            return False, f"Expected 3 parts, got {len(parts)}"
+
+        concept_part = parts[0]
+        if ":" not in concept_part:
+            return False, "Missing ':' in concept definition"
+
+        return True, None
+
+    def _load_or_create_token_cache(self, fact_file: Path) -> Dict:
+        """
+        Load token cache from .q.tokens if present and matching file hash.
+        Otherwise return a new structure with updated file-hash.
+        """
+        cache_file = fact_file.with_suffix(".q.tokens")
+        current_hash = _compute_file_hash(fact_file)
+
+        if cache_file.exists():
+            try:
+                with open(cache_file, "r") as f:
+                    cache = json.load(f)
+                # If the hash matches, return it directly
+                if cache.get("content_hash") == current_hash:
+                    return cache
+                # Otherwise, we signal that it's changed
+                self.logger.info(f"Hash changed for {fact_file}, reindex needed.")
+            except json.JSONDecodeError:
+                self.logger.warning(f"Corrupt token cache for {fact_file}, rebuilding.")
+            except Exception as e:
+                self.logger.warning(f"Error reading cache for {fact_file}: {str(e)}")
+
+        # Return a fresh cache
+        return {"facts": {}, "content_hash": current_hash}
+
+    def _save_token_cache(self, fact_file: Path, cache: Dict) -> None:
+        cache_file = fact_file.with_suffix(".q.tokens")
+        # Always ensure we're saving the correct file-hash
+        cache["content_hash"] = _compute_file_hash(fact_file)
+        with open(cache_file, "w") as f:
+            json.dump(cache, f)
+
+    def preprocess_text(self, text: str) -> List[str]:
+        parts = [x.strip() for x in text.split("|")] if "|" in text else [text]
+        # Remove : after the first word of parts[0]
+        parts[0] = re.sub(r"^(.*?):", r"\1", parts[0])
+
+        lemmatizer = WordNetLemmatizer()
+        stop_words = set(stopwords.words("english")) - {
+            "how", "what", "when", "where", "why", "which",
+        }
+
+        tokens = []
+        for part in parts:
+            if "(" in part and ")" in part:
+                code_tokens = re.findall(
+                    r'[\w_]+(?=\()|[\w_]+(?==[\'"]{1}[\w_]+[\'"]{1})', part
+                )
+                tokens.extend(code_tokens)
+
+            words = word_tokenize(part.lower())
+            tokens.extend(
+                [
+                    lemmatizer.lemmatize(token)
+                    for token in words
+                    if token not in stop_words
+                ]
+            )
+
+        return tokens
+
+    def maybe_load_bm25_index(self, clear_cache=False) -> bool:
+        """
+        Load existing BM25 index from disk, if present and clear_cache=False.
+        """
+        if not clear_cache and os.path.exists(self.bm25_index_file):
+            self.logger.info("Loading existing BM25 index from disk.")
+            with open(self.bm25_index_file, "rb") as f:
+                data = pickle.load(f)
+            self.tokenized_facts = data["tokenized_facts"]
+            self.bm25_index = data["bm25_index"]
+            return True
+        return False
+
+    def build_search_index(self, clear_cache=False) -> None:
+        """
+        Checks for new or modified .q.md files by comparing file-hash.
+        If none need reindexing and clear_cache is False, loads existing index if available.
+        Otherwise, reindexes only changed/new files and merges or creates a new index.
+        """
+        # If clear_cache is True, we skip partial logic: rebuild everything from scratch
+        if clear_cache:
+            self.logger.info("Clearing cache and rebuilding full search index.")
+            if self.bm25_index_file.exists():
+                self.bm25_index_file.unlink()
+
+        process = psutil.Process()
+        self.logger.info("Checking which .q.md files need (re)indexing...")
+
+        # Gather all .q.md files
+        q_files = [self.docs_dir / f for f in os.listdir(self.docs_dir) if f.endswith(".q.md")]
+
+        # We'll store known (unchanged) facts in these lists
+        existing_facts: List[str] = []
+        existing_tokens: List[List[str]] = []
+
+        # Keep track of invalid lines for logging
+        invalid_lines = []
+        needSet = []  # files that must be (re)indexed
+
+        for qf in q_files:
+            token_cache_file = qf.with_suffix(".q.tokens")
+
+            # If no .q.tokens or clear_cache is True → definitely reindex
+            if clear_cache or not token_cache_file.exists():
+                needSet.append(qf)
+                continue
+
+            # Otherwise, load the existing cache and compare hash
+            cache = self._load_or_create_token_cache(qf)
+            # If the .q.tokens was out of date (i.e. changed hash), we reindex
+            if len(cache["facts"]) == 0 or cache.get("content_hash") != _compute_file_hash(qf):
+                needSet.append(qf)
+            else:
+                # File is unchanged → retrieve cached token data
+                for line, cache_data in cache["facts"].items():
+                    existing_facts.append(line)
+                    existing_tokens.append(cache_data["tokens"])
+                    self.document_map[line] = qf  # track the doc for that fact
+
+        if not needSet and not clear_cache:
+            # If no file needs reindexing, try loading existing index
+            if self.maybe_load_bm25_index(clear_cache=False):
+                self.logger.info("No new/changed .q.md files found. Using existing BM25 index.")
+                return
+            else:
+                # If there's no existing index, we must build a fresh index from the old caches
+                self.logger.info("No existing BM25 index found. Building from cached facts.")
+                if existing_facts:
+                    self.logger.info(f"Building BM25 index with {len(existing_facts)} cached facts.")
+                    self.bm25_index = BM25Okapi(existing_tokens)
+                    self.tokenized_facts = existing_facts
+                    with open(self.bm25_index_file, "wb") as f:
+                        pickle.dump({
+                            "bm25_index": self.bm25_index,
+                            "tokenized_facts": self.tokenized_facts
+                        }, f)
+                else:
+                    self.logger.warning("No facts found at all. Index remains empty.")
+                return
+
+        # ----------------------------------------------------- /Users/unclecode/.crawl4ai/docs/14_proxy_security.q.q.tokens '/Users/unclecode/.crawl4ai/docs/14_proxy_security.q.md'
+        # If we reach here, we have new or changed .q.md files
+        # We'll parse them, reindex them, and then combine with existing_facts
+        # -----------------------------------------------------
+
+        self.logger.info(f"{len(needSet)} file(s) need reindexing. Parsing now...")
+
+        # 1) Parse the new or changed .q.md files
+        new_facts = []
+        new_tokens = []
+        with tqdm(total=len(needSet), desc="Indexing changed files") as file_pbar:
+            for file in needSet:
+                # We'll build up a fresh cache
+                fresh_cache = {"facts": {}, "content_hash": _compute_file_hash(file)}
+                try:
+                    with open(file, "r", encoding="utf-8") as f_obj:
+                        content = f_obj.read().strip()
+                        lines = [l.strip() for l in content.split("\n") if l.strip()]
+
+                    for line in lines:
+                        is_valid, error = self._validate_fact_line(line)
+                        if not is_valid:
+                            invalid_lines.append((file, line, error))
+                            continue
+
+                        tokens = self.preprocess_text(line)
+                        fresh_cache["facts"][line] = {
+                            "tokens": tokens,
+                            "added": time.time(),
+                        }
+                        new_facts.append(line)
+                        new_tokens.append(tokens)
+                        self.document_map[line] = file
+
+                    # Save the new .q.tokens with updated hash
+                    self._save_token_cache(file, fresh_cache)
+
+                    mem_usage = process.memory_info().rss / 1024 / 1024
+                    self.logger.debug(f"Memory usage after {file.name}: {mem_usage:.2f}MB")
+
+                except Exception as e:
+                    self.logger.error(f"Error processing {file}: {str(e)}")
+
+                file_pbar.update(1)
+
+        if invalid_lines:
+            self.logger.warning(f"Found {len(invalid_lines)} invalid fact lines:")
+            for file, line, error in invalid_lines:
+                self.logger.warning(f"{file}: {error} in line: {line[:50]}...")
+
+        # 2) Merge newly tokenized facts with the existing ones
+        all_facts = existing_facts + new_facts
+        all_tokens = existing_tokens + new_tokens
+
+        # 3) Build BM25 index from combined facts
+        self.logger.info(f"Building BM25 index with {len(all_facts)} total facts (old + new).")
+        self.bm25_index = BM25Okapi(all_tokens)
+        self.tokenized_facts = all_facts
+
+        # 4) Save the updated BM25 index to disk
+        with open(self.bm25_index_file, "wb") as f:
+            pickle.dump({
+                "bm25_index": self.bm25_index,
+                "tokenized_facts": self.tokenized_facts
+            }, f)
+
+        final_mem = process.memory_info().rss / 1024 / 1024
+        self.logger.info(f"Search index updated. Final memory usage: {final_mem:.2f}MB")
+
+    async def generate_index_files(self, force_generate_facts: bool = False, clear_bm25_cache: bool = False) -> None:
+        """
+        Generate index files for all documents in parallel batches
+        
+        Args:
+            force_generate_facts (bool): If True, regenerate indexes even if they exist
+            clear_bm25_cache (bool): If True, clear existing BM25 index cache
+        """
+        self.logger.info("Starting index generation for documentation files.")
+        
+        md_files = [
+            self.docs_dir / f for f in os.listdir(self.docs_dir) 
+            if f.endswith('.md') and not any(f.endswith(x) for x in ['.q.md', '.xs.md'])
+        ]
+
+        # Filter out files that already have .q files unless force=True
+        if not force_generate_facts:
+            md_files = [
+                f for f in md_files 
+                if not (self.docs_dir / f.name.replace('.md', '.q.md')).exists()
+            ]
+
+        if not md_files:
+            self.logger.info("All index files exist. Use force=True to regenerate.")
+        else:
+            # Process documents in batches
+            for i in range(0, len(md_files), self.batch_size):
+                batch = md_files[i:i + self.batch_size]
+                self.logger.info(f"Processing batch {i//self.batch_size + 1}/{(len(md_files)//self.batch_size) + 1}")
+                await self._process_document_batch(batch)
+
+        self.logger.info("Index generation complete, building/updating search index.")
+        self.build_search_index(clear_cache=clear_bm25_cache)
+
+    def generate(self, sections: List[str], mode: str = "extended") -> str:
+        # Get all markdown files
+        all_files = glob.glob(str(self.docs_dir / "[0-9]*.md")) + \
+                    glob.glob(str(self.docs_dir / "[0-9]*.xs.md"))
+        
+        # Extract base names without extensions
+        base_docs = {Path(f).name.split('.')[0] for f in all_files 
+                        if not Path(f).name.endswith('.q.md')}
+        
+        # Filter by sections if provided
+        if sections:
+            base_docs = {doc for doc in base_docs 
+                        if any(section.lower() in doc.lower() for section in sections)}
+        
+        # Get file paths based on mode
+        files = []
+        for doc in sorted(base_docs, key=lambda x: int(x.split('_')[0]) if x.split('_')[0].isdigit() else 999999):
+            if mode == "condensed":
+                xs_file = self.docs_dir / f"{doc}.xs.md"
+                regular_file = self.docs_dir / f"{doc}.md"
+                files.append(str(xs_file if xs_file.exists() else regular_file))
+            else:
+                files.append(str(self.docs_dir / f"{doc}.md"))
+
+        # Read and format content
+        content = []
+        for file in files:
+            try:
+                with open(file, 'r', encoding='utf-8') as f:
+                    fname = Path(file).name
+                    content.append(f"{'#'*20}\n# {fname}\n{'#'*20}\n\n{f.read()}")
+            except Exception as e:
+                self.logger.error(f"Error reading {file}: {str(e)}")
+
+        return "\n\n---\n\n".join(content) if content else ""
+
+    def search(self, query: str, top_k: int = 5) -> str:
+        if not self.bm25_index:
+            return "No search index available. Call build_search_index() first."
+
+        query_tokens = self.preprocess_text(query)
+        doc_scores = self.bm25_index.get_scores(query_tokens)
+
+        mean_score = np.mean(doc_scores)
+        std_score = np.std(doc_scores)
+        score_threshold = mean_score + (0.25 * std_score)
+
+        file_data = self._aggregate_search_scores(
+            doc_scores=doc_scores,
+            score_threshold=score_threshold,
+            query_tokens=query_tokens,
+        )
+
+        ranked_files = sorted(
+            file_data.items(),
+            key=lambda x: (
+                x[1]["code_match_score"] * 2.0
+                + x[1]["match_count"] * 1.5
+                + x[1]["total_score"]
+            ),
+            reverse=True,
+        )[:top_k]
+
+        results = []
+        for file, _ in ranked_files:
+            main_doc = str(file).replace(".q.md", ".md")
+            if os.path.exists(self.docs_dir / main_doc):
+                with open(self.docs_dir / main_doc, "r", encoding='utf-8') as f:
+                    only_file_name = main_doc.split("/")[-1]
+                    content = [
+                    "#" * 20,
+                    f"# {only_file_name}",
+                    "#" * 20,
+                    "",
+                    f.read()
+                    ]
+                    results.append("\n".join(content))
+
+        return "\n\n---\n\n".join(results)
+
+    def _aggregate_search_scores(
+        self, doc_scores: List[float], score_threshold: float, query_tokens: List[str]
+    ) -> Dict:
+        file_data = {}
+
+        for idx, score in enumerate(doc_scores):
+            if score <= score_threshold:
+                continue
+
+            fact = self.tokenized_facts[idx]
+            file_path = self.document_map[fact]
+
+            if file_path not in file_data:
+                file_data[file_path] = {
+                    "total_score": 0,
+                    "match_count": 0,
+                    "code_match_score": 0,
+                    "matched_facts": [],
+                }
+
+            components = fact.split("|") if "|" in fact else [fact]
+
+            code_match_score = 0
+            if len(components) == 3:
+                code_ref = components[2].strip()
+                code_tokens = self.preprocess_text(code_ref)
+                code_match_score = len(set(query_tokens) & set(code_tokens)) / len(query_tokens)
+
+            file_data[file_path]["total_score"] += score
+            file_data[file_path]["match_count"] += 1
+            file_data[file_path]["code_match_score"] = max(
+                file_data[file_path]["code_match_score"], code_match_score
+            )
+            file_data[file_path]["matched_facts"].append(fact)
+
+        return file_data
+
+    def refresh_index(self) -> None:
+        """Convenience method for a full rebuild."""
+        self.build_search_index(clear_cache=True)

crawl4ai/markdown_generation_strategy.py

@@ -0,0 +1,225 @@
+from abc import ABC, abstractmethod
+from typing import Optional, Dict, Any, Tuple
+from .models import MarkdownGenerationResult
+from .html2text import CustomHTML2Text
+from .content_filter_strategy import RelevantContentFilter, BM25ContentFilter
+import re
+from urllib.parse import urljoin
+
+# Pre-compile the regex pattern
+LINK_PATTERN = re.compile(r'!?\[([^\]]+)\]\(([^)]+?)(?:\s+"([^"]*)")?\)')
+
+def fast_urljoin(base: str, url: str) -> str:
+    """Fast URL joining for common cases."""
+    if url.startswith(('http://', 'https://', 'mailto:', '//')):
+        return url
+    if url.startswith('/'):
+        # Handle absolute paths
+        if base.endswith('/'):
+            return base[:-1] + url
+        return base + url
+    return urljoin(base, url)
+
+class MarkdownGenerationStrategy(ABC):
+    """Abstract base class for markdown generation strategies."""
+    def __init__(self, content_filter: Optional[RelevantContentFilter] = None, options: Optional[Dict[str, Any]] = None):
+        self.content_filter = content_filter
+        self.options = options or {}
+    
+    @abstractmethod
+    def generate_markdown(self, 
+                         cleaned_html: str, 
+                         base_url: str = "",
+                         html2text_options: Optional[Dict[str, Any]] = None,
+                         content_filter: Optional[RelevantContentFilter] = None,
+                         citations: bool = True,
+                         **kwargs) -> MarkdownGenerationResult:
+        """Generate markdown from cleaned HTML."""
+        pass
+
+class DefaultMarkdownGenerator(MarkdownGenerationStrategy):
+    """
+    Default implementation of markdown generation strategy.
+    
+    How it works:
+    1. Generate raw markdown from cleaned HTML.
+    2. Convert links to citations.
+    3. Generate fit markdown if content filter is provided.
+    4. Return MarkdownGenerationResult.
+    
+    Args:
+        content_filter (Optional[RelevantContentFilter]): Content filter for generating fit markdown.
+        options (Optional[Dict[str, Any]]): Additional options for markdown generation. Defaults to None.
+        
+    Returns:
+        MarkdownGenerationResult: Result containing raw markdown, fit markdown, fit HTML, and references markdown.
+    """
+    def __init__(self, content_filter: Optional[RelevantContentFilter] = None, options: Optional[Dict[str, Any]] = None):
+        super().__init__(content_filter, options)
+    
+    def convert_links_to_citations(self, markdown: str, base_url: str = "") -> Tuple[str, str]:
+        """
+        Convert links in markdown to citations.
+        
+        How it works:
+        1. Find all links in the markdown.
+        2. Convert links to citations.
+        3. Return converted markdown and references markdown.
+        
+        Note:
+        This function uses a regex pattern to find links in markdown.
+        
+        Args:
+            markdown (str): Markdown text.
+            base_url (str): Base URL for URL joins.
+            
+        Returns:
+            Tuple[str, str]: Converted markdown and references markdown.
+        """
+        link_map = {}
+        url_cache = {}  # Cache for URL joins
+        parts = []
+        last_end = 0
+        counter = 1
+        
+        for match in LINK_PATTERN.finditer(markdown):
+            parts.append(markdown[last_end:match.start()])
+            text, url, title = match.groups()
+            
+            # Use cached URL if available, otherwise compute and cache
+            if base_url and not url.startswith(('http://', 'https://', 'mailto:')):
+                if url not in url_cache:
+                    url_cache[url] = fast_urljoin(base_url, url)
+                url = url_cache[url]
+                
+            if url not in link_map:
+                desc = []
+                if title: desc.append(title)
+                if text and text != title: desc.append(text)
+                link_map[url] = (counter, ": " + " - ".join(desc) if desc else "")
+                counter += 1
+                
+            num = link_map[url][0]
+            parts.append(f"{text}⟨{num}⟩" if not match.group(0).startswith('!') else f"![{text}⟨{num}⟩]")
+            last_end = match.end()
+        
+        parts.append(markdown[last_end:])
+        converted_text = ''.join(parts)
+        
+        # Pre-build reference strings
+        references = ["\n\n## References\n\n"]
+        references.extend(
+            f"⟨{num}⟩ {url}{desc}\n" 
+            for url, (num, desc) in sorted(link_map.items(), key=lambda x: x[1][0])
+        )
+        
+        return converted_text, ''.join(references)
+
+    def generate_markdown(self, 
+                         cleaned_html: str, 
+                         base_url: str = "",
+                         html2text_options: Optional[Dict[str, Any]] = None,
+                         options: Optional[Dict[str, Any]] = None,
+                         content_filter: Optional[RelevantContentFilter] = None,
+                         citations: bool = True,
+                         **kwargs) -> MarkdownGenerationResult:
+        """
+        Generate markdown with citations from cleaned HTML.
+        
+        How it works:
+        1. Generate raw markdown from cleaned HTML.
+        2. Convert links to citations.
+        3. Generate fit markdown if content filter is provided.
+        4. Return MarkdownGenerationResult.
+        
+        Args:
+            cleaned_html (str): Cleaned HTML content.
+            base_url (str): Base URL for URL joins.
+            html2text_options (Optional[Dict[str, Any]]): HTML2Text options.
+            options (Optional[Dict[str, Any]]): Additional options for markdown generation.
+            content_filter (Optional[RelevantContentFilter]): Content filter for generating fit markdown.
+            citations (bool): Whether to generate citations.
+            
+        Returns:
+            MarkdownGenerationResult: Result containing raw markdown, fit markdown, fit HTML, and references markdown.
+        """
+        try:
+            # Initialize HTML2Text with default options for better conversion
+            h = CustomHTML2Text(baseurl=base_url)
+            default_options = {
+                'body_width': 0,  # Disable text wrapping
+                'ignore_emphasis': False,
+                'ignore_links': False,
+                'ignore_images': False,
+                'protect_links': True,
+                'single_line_break': True,
+                'mark_code': True,
+                'escape_snob': False
+            }
+            
+            # Update with custom options if provided
+            if html2text_options:
+                default_options.update(html2text_options)
+            elif options:
+                default_options.update(options)
+            elif self.options:
+                default_options.update(self.options)
+            
+            h.update_params(**default_options)
+
+            # Ensure we have valid input
+            if not cleaned_html:
+                cleaned_html = ""
+            elif not isinstance(cleaned_html, str):
+                cleaned_html = str(cleaned_html)
+
+            # Generate raw markdown
+            try:
+                raw_markdown = h.handle(cleaned_html)
+            except Exception as e:
+                raw_markdown = f"Error converting HTML to markdown: {str(e)}"
+            
+            raw_markdown = raw_markdown.replace('    ```', '```')
+
+            # Convert links to citations
+            markdown_with_citations: str = raw_markdown
+            references_markdown: str = ""
+            if citations:
+                try:
+                    markdown_with_citations, references_markdown = self.convert_links_to_citations(
+                        raw_markdown, base_url
+                    )
+                except Exception as e:
+                    markdown_with_citations = raw_markdown
+                    references_markdown = f"Error generating citations: {str(e)}"
+
+            # Generate fit markdown if content filter is provided
+            fit_markdown: Optional[str] = ""
+            filtered_html: Optional[str] = ""
+            if content_filter or self.content_filter:
+                try:
+                    content_filter = content_filter or self.content_filter
+                    filtered_html = content_filter.filter_content(cleaned_html)
+                    filtered_html = '\n'.join('<div>{}</div>'.format(s) for s in filtered_html)
+                    fit_markdown = h.handle(filtered_html)
+                except Exception as e:
+                    fit_markdown = f"Error generating fit markdown: {str(e)}"
+                    filtered_html = ""
+
+            return MarkdownGenerationResult(
+                raw_markdown=raw_markdown or "",
+                markdown_with_citations=markdown_with_citations or "",
+                references_markdown=references_markdown or "",
+                fit_markdown=fit_markdown or "",
+                fit_html=filtered_html or "",
+            )
+        except Exception as e:
+            # If anything fails, return empty strings with error message
+            error_msg = f"Error in markdown generation: {str(e)}"
+            return MarkdownGenerationResult(
+                raw_markdown=error_msg,
+                markdown_with_citations=error_msg,
+                references_markdown="",
+                fit_markdown="",
+                fit_html="",
+            )

crawl4ai/migrations.py

@@ -0,0 +1,168 @@
+import os
+import asyncio
+import logging
+from pathlib import Path
+import aiosqlite
+from typing import Optional
+import xxhash
+import aiofiles
+import shutil
+import time
+from datetime import datetime
+from .async_logger import AsyncLogger, LogLevel
+
+# Initialize logger
+logger = AsyncLogger(log_level=LogLevel.DEBUG, verbose=True)
+
+# logging.basicConfig(level=logging.INFO)
+# logger = logging.getLogger(__name__)
+
+class DatabaseMigration:
+    def __init__(self, db_path: str):
+        self.db_path = db_path
+        self.content_paths = self._ensure_content_dirs(os.path.dirname(db_path))
+        
+    def _ensure_content_dirs(self, base_path: str) -> dict:
+        dirs = {
+            'html': 'html_content',
+            'cleaned': 'cleaned_html',
+            'markdown': 'markdown_content', 
+            'extracted': 'extracted_content',
+            'screenshots': 'screenshots'
+        }
+        content_paths = {}
+        for key, dirname in dirs.items():
+            path = os.path.join(base_path, dirname)
+            os.makedirs(path, exist_ok=True)
+            content_paths[key] = path
+        return content_paths
+
+    def _generate_content_hash(self, content: str) -> str:
+        x = xxhash.xxh64()
+        x.update(content.encode())
+        content_hash = x.hexdigest()
+        return content_hash
+        # return hashlib.sha256(content.encode()).hexdigest()
+
+    async def _store_content(self, content: str, content_type: str) -> str:
+        if not content:
+            return ""
+        
+        content_hash = self._generate_content_hash(content)
+        file_path = os.path.join(self.content_paths[content_type], content_hash)
+        
+        if not os.path.exists(file_path):
+            async with aiofiles.open(file_path, 'w', encoding='utf-8') as f:
+                await f.write(content)
+                
+        return content_hash
+
+    async def migrate_database(self):
+        """Migrate existing database to file-based storage"""
+        # logger.info("Starting database migration...")
+        logger.info("Starting database migration...", tag="INIT")
+        
+        try:
+            async with aiosqlite.connect(self.db_path) as db:
+                # Get all rows
+                async with db.execute(
+                    '''SELECT url, html, cleaned_html, markdown, 
+                       extracted_content, screenshot FROM crawled_data'''
+                ) as cursor:
+                    rows = await cursor.fetchall()
+
+                migrated_count = 0
+                for row in rows:
+                    url, html, cleaned_html, markdown, extracted_content, screenshot = row
+                    
+                    # Store content in files and get hashes
+                    html_hash = await self._store_content(html, 'html')
+                    cleaned_hash = await self._store_content(cleaned_html, 'cleaned')
+                    markdown_hash = await self._store_content(markdown, 'markdown')
+                    extracted_hash = await self._store_content(extracted_content, 'extracted')
+                    screenshot_hash = await self._store_content(screenshot, 'screenshots')
+
+                    # Update database with hashes
+                    await db.execute('''
+                        UPDATE crawled_data 
+                        SET html = ?, 
+                            cleaned_html = ?,
+                            markdown = ?,
+                            extracted_content = ?,
+                            screenshot = ?
+                        WHERE url = ?
+                    ''', (html_hash, cleaned_hash, markdown_hash, 
+                         extracted_hash, screenshot_hash, url))
+                    
+                    migrated_count += 1
+                    if migrated_count % 100 == 0:
+                        logger.info(f"Migrated {migrated_count} records...", tag="INIT")
+                        
+
+                await db.commit()
+                logger.success(f"Migration completed. {migrated_count} records processed.", tag="COMPLETE")
+
+        except Exception as e:
+            # logger.error(f"Migration failed: {e}")
+            logger.error(
+                message="Migration failed: {error}",
+                tag="ERROR",
+                params={"error": str(e)}
+            )
+            raise e
+
+async def backup_database(db_path: str) -> str:
+    """Create backup of existing database"""
+    if not os.path.exists(db_path):
+        logger.info("No existing database found. Skipping backup.", tag="INIT")
+        return None
+        
+    # Create backup with timestamp
+    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+    backup_path = f"{db_path}.backup_{timestamp}"
+    
+    try:
+        # Wait for any potential write operations to finish
+        await asyncio.sleep(1)
+        
+        # Create backup
+        shutil.copy2(db_path, backup_path)
+        logger.info(f"Database backup created at: {backup_path}", tag="COMPLETE")
+        return backup_path
+    except Exception as e:
+        # logger.error(f"Backup failed: {e}")
+        logger.error(
+                message="Migration failed: {error}",
+                tag="ERROR",
+                params={"error": str(e)}
+            )
+        raise e
+    
+async def run_migration(db_path: Optional[str] = None):
+    """Run database migration"""
+    if db_path is None:
+        db_path = os.path.join(Path.home(), ".crawl4ai", "crawl4ai.db")
+    
+    if not os.path.exists(db_path):
+        logger.info("No existing database found. Skipping migration.", tag="INIT")
+        return
+        
+    # Create backup first
+    backup_path = await backup_database(db_path)
+    if not backup_path:
+        return
+    
+    migration = DatabaseMigration(db_path)
+    await migration.migrate_database()
+    
+def main():
+    """CLI entry point for migration"""
+    import argparse
+    parser = argparse.ArgumentParser(description='Migrate Crawl4AI database to file-based storage')
+    parser.add_argument('--db-path', help='Custom database path')
+    args = parser.parse_args()
+    
+    asyncio.run(run_migration(args.db_path))
+
+if __name__ == "__main__":
+    main()

crawl4ai/model_loader.py

@@ -0,0 +1,256 @@
+from functools import lru_cache
+from pathlib import Path
+import subprocess, os
+import shutil
+import tarfile
+from .model_loader import *
+import argparse
+import urllib.request
+from crawl4ai.config import MODEL_REPO_BRANCH
+__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+
+@lru_cache()
+def get_available_memory(device):
+    import torch
+    if device.type == 'cuda':
+        return torch.cuda.get_device_properties(device).total_memory
+    elif device.type == 'mps':      
+        return 48 * 1024 ** 3  # Assuming 8GB for MPS, as a conservative estimate
+    else:
+        return 0
+
+@lru_cache()
+def calculate_batch_size(device):
+    available_memory = get_available_memory(device)
+    
+    if device.type == 'cpu':
+        return 16
+    elif device.type in ['cuda', 'mps']:
+        # Adjust these thresholds based on your model size and available memory
+        if available_memory >= 31 * 1024 ** 3:  # > 32GB
+            return 256
+        elif available_memory >= 15 * 1024 ** 3:  # > 16GB to 32GB
+            return 128
+        elif available_memory >= 8 * 1024 ** 3:  # 8GB to 16GB
+            return 64
+        else:
+            return 32
+    else:
+        return 16  # Default batch size   
+    
+@lru_cache()
+def get_device():
+    import torch
+    if torch.cuda.is_available():
+        device = torch.device('cuda')
+    elif torch.backends.mps.is_available():
+        device = torch.device('mps')
+    else:
+        device = torch.device('cpu')
+    return device   
+    
+def set_model_device(model):
+    device = get_device()
+    model.to(device)    
+    return model, device
+
+@lru_cache()
+def get_home_folder():
+    home_folder = os.path.join(os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai")
+    os.makedirs(home_folder, exist_ok=True)
+    os.makedirs(f"{home_folder}/cache", exist_ok=True)
+    os.makedirs(f"{home_folder}/models", exist_ok=True)
+    return home_folder 
+
+@lru_cache()
+def load_bert_base_uncased():
+    from transformers import BertTokenizer, BertModel, AutoTokenizer, AutoModel
+    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', resume_download=None)
+    model = BertModel.from_pretrained('bert-base-uncased', resume_download=None)
+    model.eval()
+    model, device = set_model_device(model)
+    return tokenizer, model
+
+@lru_cache()
+def load_HF_embedding_model(model_name="BAAI/bge-small-en-v1.5") -> tuple:
+    """Load the Hugging Face model for embedding.
+    
+    Args:
+        model_name (str, optional): The model name to load. Defaults to "BAAI/bge-small-en-v1.5".
+        
+    Returns:
+        tuple: The tokenizer and model.
+    """
+    from transformers import BertTokenizer, BertModel, AutoTokenizer, AutoModel
+    tokenizer = AutoTokenizer.from_pretrained(model_name, resume_download=None)
+    model = AutoModel.from_pretrained(model_name, resume_download=None)
+    model.eval()
+    model, device = set_model_device(model)
+    return tokenizer, model
+
+@lru_cache()
+def load_text_classifier():
+    from transformers import AutoTokenizer, AutoModelForSequenceClassification
+    from transformers import pipeline
+    import torch
+
+    tokenizer = AutoTokenizer.from_pretrained("dstefa/roberta-base_topic_classification_nyt_news")
+    model = AutoModelForSequenceClassification.from_pretrained("dstefa/roberta-base_topic_classification_nyt_news")
+    model.eval()
+    model, device = set_model_device(model)
+    pipe = pipeline("text-classification", model=model, tokenizer=tokenizer)
+    return pipe
+
+@lru_cache()
+def load_text_multilabel_classifier():
+    from transformers import AutoModelForSequenceClassification, AutoTokenizer
+    import numpy as np
+    from scipy.special import expit
+    import torch
+
+    # # Check for available device: CUDA, MPS (for Apple Silicon), or CPU
+    # if torch.cuda.is_available():
+    #     device = torch.device("cuda")
+    # elif torch.backends.mps.is_available():
+    #     device = torch.device("mps")
+    # else:
+    #     device = torch.device("cpu")
+    #     # return load_spacy_model(), torch.device("cpu")
+    
+
+    MODEL = "cardiffnlp/tweet-topic-21-multi"
+    tokenizer = AutoTokenizer.from_pretrained(MODEL, resume_download=None)
+    model = AutoModelForSequenceClassification.from_pretrained(MODEL, resume_download=None)
+    model.eval()
+    model, device = set_model_device(model)
+    class_mapping = model.config.id2label
+
+    def _classifier(texts, threshold=0.5, max_length=64):
+        tokens = tokenizer(texts, return_tensors='pt', padding=True, truncation=True, max_length=max_length)
+        tokens = {key: val.to(device) for key, val in tokens.items()}  # Move tokens to the selected device
+
+        with torch.no_grad():
+            output = model(**tokens)
+
+        scores = output.logits.detach().cpu().numpy()
+        scores = expit(scores)
+        predictions = (scores >= threshold) * 1
+
+        batch_labels = []
+        for prediction in predictions:
+            labels = [class_mapping[i] for i, value in enumerate(prediction) if value == 1]
+            batch_labels.append(labels)
+
+        return batch_labels
+
+    return _classifier, device
+
+@lru_cache()
+def load_nltk_punkt():
+    import nltk
+    try:
+        nltk.data.find('tokenizers/punkt')
+    except LookupError:
+        nltk.download('punkt')
+    return nltk.data.find('tokenizers/punkt')
+
+@lru_cache()
+def load_spacy_model():
+    import spacy
+    name = "models/reuters"
+    home_folder = get_home_folder()
+    model_folder = Path(home_folder) / name
+    
+    # Check if the model directory already exists
+    if not (model_folder.exists() and any(model_folder.iterdir())):
+        repo_url = "https://github.com/unclecode/crawl4ai.git"
+        branch = MODEL_REPO_BRANCH 
+        repo_folder = Path(home_folder) / "crawl4ai"
+        
+        print("[LOG] ⏬ Downloading Spacy model for the first time...")
+
+        # Remove existing repo folder if it exists
+        if repo_folder.exists():
+            try:
+                shutil.rmtree(repo_folder)
+                if model_folder.exists():
+                    shutil.rmtree(model_folder)
+            except PermissionError:
+                print("[WARNING] Unable to remove existing folders. Please manually delete the following folders and try again:")
+                print(f"- {repo_folder}")
+                print(f"- {model_folder}")
+                return None
+
+        try:
+            # Clone the repository
+            subprocess.run(
+                ["git", "clone", "-b", branch, repo_url, str(repo_folder)],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                check=True
+            )
+
+            # Create the models directory if it doesn't exist
+            models_folder = Path(home_folder) / "models"
+            models_folder.mkdir(parents=True, exist_ok=True)
+
+            # Copy the reuters model folder to the models directory
+            source_folder = repo_folder / "models" / "reuters"
+            shutil.copytree(source_folder, model_folder)
+
+            # Remove the cloned repository
+            shutil.rmtree(repo_folder)
+
+            print("[LOG] ✅ Spacy Model downloaded successfully")
+        except subprocess.CalledProcessError as e:
+            print(f"An error occurred while cloning the repository: {e}")
+            return None
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            return None
+
+    try:
+        return spacy.load(str(model_folder))
+    except Exception as e:
+        print(f"Error loading spacy model: {e}")
+        return None
+
+def download_all_models(remove_existing=False):
+    """Download all models required for Crawl4AI."""
+    if remove_existing:
+        print("[LOG] Removing existing models...")
+        home_folder = get_home_folder()
+        model_folders = [
+            os.path.join(home_folder, "models/reuters"),
+            os.path.join(home_folder, "models"),
+        ]
+        for folder in model_folders:
+            if Path(folder).exists():
+                shutil.rmtree(folder)
+        print("[LOG] Existing models removed.")
+
+    # Load each model to trigger download
+    # print("[LOG] Downloading BERT Base Uncased...")
+    # load_bert_base_uncased()
+    # print("[LOG] Downloading BGE Small EN v1.5...")
+    # load_bge_small_en_v1_5()
+    # print("[LOG] Downloading ONNX model...")
+    # load_onnx_all_MiniLM_l6_v2()
+    print("[LOG] Downloading text classifier...")
+    _, device = load_text_multilabel_classifier()
+    print(f"[LOG] Text classifier loaded on {device}")
+    print("[LOG] Downloading custom NLTK Punkt model...")
+    load_nltk_punkt()
+    print("[LOG] ✅ All models downloaded successfully.")
+
+def main():
+    print("[LOG] Welcome to the Crawl4AI Model Downloader!")
+    print("[LOG] This script will download all the models required for Crawl4AI.")
+    parser = argparse.ArgumentParser(description="Crawl4AI Model Downloader")
+    parser.add_argument('--remove-existing', action='store_true', help="Remove existing models before downloading")
+    args = parser.parse_args()
+    
+    download_all_models(remove_existing=args.remove_existing)
+
+if __name__ == "__main__":
+    main()

crawl4ai/models.py

@@ -0,0 +1,61 @@
+from pydantic import BaseModel, HttpUrl
+from typing import List, Dict, Optional, Callable, Awaitable, Union, Any
+from dataclasses import dataclass
+from .ssl_certificate import SSLCertificate
+
+@dataclass
+class TokenUsage:
+    completion_tokens: int = 0
+    prompt_tokens: int = 0 
+    total_tokens: int = 0
+    completion_tokens_details: Optional[dict] = None
+    prompt_tokens_details: Optional[dict] = None
+    
+
+class UrlModel(BaseModel):
+    url: HttpUrl
+    forced: bool = False
+
+class MarkdownGenerationResult(BaseModel):
+    raw_markdown: str
+    markdown_with_citations: str
+    references_markdown: str
+    fit_markdown: Optional[str] = None
+    fit_html: Optional[str] = None
+
+class CrawlResult(BaseModel):
+    url: str
+    html: str
+    success: bool
+    cleaned_html: Optional[str] = None
+    media: Dict[str, List[Dict]] = {}
+    links: Dict[str, List[Dict]] = {}
+    downloaded_files: Optional[List[str]] = None
+    screenshot: Optional[str] = None
+    pdf : Optional[bytes] = None
+    markdown: Optional[Union[str, MarkdownGenerationResult]] = None
+    markdown_v2: Optional[MarkdownGenerationResult] = None
+    fit_markdown: Optional[str] = None
+    fit_html: Optional[str] = None
+    extracted_content: Optional[str] = None
+    metadata: Optional[dict] = None
+    error_message: Optional[str] = None
+    session_id: Optional[str] = None
+    response_headers: Optional[dict] = None
+    status_code: Optional[int] = None
+    ssl_certificate: Optional[SSLCertificate] = None
+    class Config:
+        arbitrary_types_allowed = True
+
+class AsyncCrawlResponse(BaseModel):
+    html: str
+    response_headers: Dict[str, str]
+    status_code: int
+    screenshot: Optional[str] = None
+    pdf_data: Optional[bytes] = None
+    get_delayed_content: Optional[Callable[[Optional[float]], Awaitable[str]]] = None
+    downloaded_files: Optional[List[str]] = None
+    ssl_certificate: Optional[SSLCertificate] = None
+
+    class Config:
+        arbitrary_types_allowed = True

crawl4ai/prompts.py

@@ -0,0 +1,204 @@
+PROMPT_EXTRACT_BLOCKS = """Here is the URL of the webpage:
+<url>{URL}</url>
+
+And here is the cleaned HTML content of that webpage:
+<html>
+{HTML}
+</html>
+
+Your task is to break down this HTML content into semantically relevant blocks, and for each block, generate a JSON object with the following keys:
+
+- index: an integer representing the index of the block in the content
+- tags: a list of semantic tags that are relevant to the content of the block
+- content: a list of strings containing the text content of the block
+- questions: a list of 3 questions that a user may ask about the content in this block
+
+To generate the JSON objects:
+
+1. Carefully read through the HTML content and identify logical breaks or shifts in the content that would warrant splitting it into separate blocks.
+
+2. For each block:
+   a. Assign it an index based on its order in the content.
+   b. Analyze the content and generate a list of relevant semantic tags that describe what the block is about.
+   c. Extract the text content, clean it up if needed, and store it as a list of strings in the "content" field.
+   d. Come up with 3 questions that a user might ask about this specific block of content, based on the tags and content. The questions should be relevant and answerable by the content in the block.
+
+3. Ensure that the order of the JSON objects matches the order of the blocks as they appear in the original HTML content.
+
+4. Double-check that each JSON object includes all required keys (index, tags, content, questions) and that the values are in the expected format (integer, list of strings, etc.).
+
+5. Make sure the generated JSON is complete and parsable, with no errors or omissions.
+
+6. Make sure to escape any special characters in the HTML content, and also single or double quote to avoid JSON parsing issues.
+
+Please provide your output within <blocks> tags, like this:
+
+<blocks>
+[{
+  "index": 0,
+  "tags": ["introduction", "overview"],
+  "content": ["This is the first paragraph of the article, which provides an introduction and overview of the main topic."],
+  "questions": [
+    "What is the main topic of this article?",
+    "What can I expect to learn from reading this article?",
+    "Is this article suitable for beginners or experts in the field?"
+  ]
+},
+{
+  "index": 1,
+  "tags": ["history", "background"],
+  "content": ["This is the second paragraph, which delves into the history and background of the topic.",
+              "It provides context and sets the stage for the rest of the article."],
+  "questions": [
+    "What historical events led to the development of this topic?",
+    "How has the understanding of this topic evolved over time?",
+    "What are some key milestones in the history of this topic?"
+  ]
+}]
+</blocks>
+
+Remember, the output should be a complete, parsable JSON wrapped in <blocks> tags, with no omissions or errors. The JSON objects should semantically break down the content into relevant blocks, maintaining the original order."""
+
+PROMPT_EXTRACT_BLOCKS = """Here is the URL of the webpage:
+<url>{URL}</url>
+
+And here is the cleaned HTML content of that webpage:
+<html>
+{HTML}
+</html>
+
+Your task is to break down this HTML content into semantically relevant blocks, and for each block, generate a JSON object with the following keys:
+
+- index: an integer representing the index of the block in the content
+- content: a list of strings containing the text content of the block
+
+To generate the JSON objects:
+
+1. Carefully read through the HTML content and identify logical breaks or shifts in the content that would warrant splitting it into separate blocks.
+
+2. For each block:
+   a. Assign it an index based on its order in the content.
+   b. Analyze the content and generate ONE semantic tag that describe what the block is about.
+   c. Extract the text content, EXACTLY SAME AS THE GIVE DATA, clean it up if needed, and store it as a list of strings in the "content" field.
+
+3. Ensure that the order of the JSON objects matches the order of the blocks as they appear in the original HTML content.
+
+4. Double-check that each JSON object includes all required keys (index, tag, content) and that the values are in the expected format (integer, list of strings, etc.).
+
+5. Make sure the generated JSON is complete and parsable, with no errors or omissions.
+
+6. Make sure to escape any special characters in the HTML content, and also single or double quote to avoid JSON parsing issues.
+
+7. Never alter the extracted content, just copy and paste it as it is.
+
+Please provide your output within <blocks> tags, like this:
+
+<blocks>
+[{
+  "index": 0,
+  "tags": ["introduction"],
+  "content": ["This is the first paragraph of the article, which provides an introduction and overview of the main topic."]
+},
+{
+  "index": 1,
+  "tags": ["background"],
+  "content": ["This is the second paragraph, which delves into the history and background of the topic.",
+              "It provides context and sets the stage for the rest of the article."]
+}]
+</blocks>
+
+Remember, the output should be a complete, parsable JSON wrapped in <blocks> tags, with no omissions or errors. The JSON objects should semantically break down the content into relevant blocks, maintaining the original order."""
+
+PROMPT_EXTRACT_BLOCKS_WITH_INSTRUCTION = """Here is the URL of the webpage:
+<url>{URL}</url>
+
+And here is the cleaned HTML content of that webpage:
+<html>
+{HTML}
+</html>
+
+Your task is to break down this HTML content into semantically relevant blocks, following the provided user's REQUEST, and for each block, generate a JSON object with the following keys:
+
+- index: an integer representing the index of the block in the content
+- content: a list of strings containing the text content of the block
+
+This is the user's REQUEST, pay attention to it:
+<request>
+{REQUEST}
+</request>
+
+To generate the JSON objects:
+
+1. Carefully read through the HTML content and identify logical breaks or shifts in the content that would warrant splitting it into separate blocks.
+
+2. For each block:
+   a. Assign it an index based on its order in the content.
+   b. Analyze the content and generate ONE semantic tag that describe what the block is about.
+   c. Extract the text content, EXACTLY SAME AS GIVE DATA, clean it up if needed, and store it as a list of strings in the "content" field.
+
+3. Ensure that the order of the JSON objects matches the order of the blocks as they appear in the original HTML content.
+
+4. Double-check that each JSON object includes all required keys (index, tag, content) and that the values are in the expected format (integer, list of strings, etc.).
+
+5. Make sure the generated JSON is complete and parsable, with no errors or omissions.
+
+6. Make sure to escape any special characters in the HTML content, and also single or double quote to avoid JSON parsing issues.
+
+7. Never alter the extracted content, just copy and paste it as it is.
+
+Please provide your output within <blocks> tags, like this:
+
+<blocks>
+[{
+  "index": 0,
+  "tags": ["introduction"],
+  "content": ["This is the first paragraph of the article, which provides an introduction and overview of the main topic."]
+},
+{
+  "index": 1,
+  "tags": ["background"],
+  "content": ["This is the second paragraph, which delves into the history and background of the topic.",
+              "It provides context and sets the stage for the rest of the article."]
+}]
+</blocks>
+
+**Make sure to follow the user instruction to extract blocks aligin with the instruction.**
+
+Remember, the output should be a complete, parsable JSON wrapped in <blocks> tags, with no omissions or errors. The JSON objects should semantically break down the content into relevant blocks, maintaining the original order."""
+
+PROMPT_EXTRACT_SCHEMA_WITH_INSTRUCTION = """Here is the content from the URL:
+<url>{URL}</url>
+
+<url_content>
+{HTML}
+</url_content>
+
+The user has made the following request for what information to extract from the above content:
+
+<user_request>
+{REQUEST}
+</user_request>
+
+<schema_block>
+{SCHEMA}
+</schema_block>
+
+Please carefully read the URL content and the user's request. If the user provided a desired JSON schema in the <schema_block> above, extract the requested information from the URL content according to that schema. If no schema was provided, infer an appropriate JSON schema based on the user's request that will best capture the key information they are looking for.
+
+Extraction instructions:
+Return the extracted information as a list of JSON objects, with each object in the list corresponding to a block of content from the URL, in the same order as it appears on the page. Wrap the entire JSON list in <blocks>...</blocks> XML tags.
+
+Quality Reflection:
+Before outputting your final answer, double check that the JSON you are returning is complete, containing all the information requested by the user, and is valid JSON that could be parsed by json.loads() with no errors or omissions. The outputted JSON objects should fully match the schema, either provided or inferred.
+
+Quality Score:
+After reflecting, score the quality and completeness of the JSON data you are about to return on a scale of 1 to 5. Write the score inside <score> tags.
+
+Avoid Common Mistakes:
+- Do NOT add any comments using "//" or "#" in the JSON output. It causes parsing errors.
+- Make sure the JSON is properly formatted with curly braces, square brackets, and commas in the right places.
+- Do not miss closing </blocks> tag at the end of the JSON output.
+- Do not generate the Python coee show me how to do the task, this is your task to extract the information and return it in JSON format.
+
+Result
+Output the final list of JSON objects, wrapped in <blocks>...</blocks> XML tags. Make sure to close the tag properly."""

crawl4ai/ssl_certificate.py

@@ -0,0 +1,181 @@
+"""SSL Certificate class for handling certificate operations."""
+
+import ssl
+import socket
+import base64
+import json
+from typing import Dict, Any, Optional
+from urllib.parse import urlparse
+import OpenSSL.crypto
+from pathlib import Path
+
+
+class SSLCertificate:
+    """
+    A class representing an SSL certificate with methods to export in various formats.
+    
+    Attributes:
+        cert_info (Dict[str, Any]): The certificate information.
+        
+        Methods:
+            from_url(url: str, timeout: int = 10) -> Optional['SSLCertificate']: Create SSLCertificate instance from a URL.
+            from_file(file_path: str) -> Optional['SSLCertificate']: Create SSLCertificate instance from a file.
+            from_binary(binary_data: bytes) -> Optional['SSLCertificate']: Create SSLCertificate instance from binary data.
+            export_as_pem() -> str: Export the certificate as PEM format.
+            export_as_der() -> bytes: Export the certificate as DER format.
+            export_as_json() -> Dict[str, Any]: Export the certificate as JSON format.
+            export_as_text() -> str: Export the certificate as text format.
+    """
+    def __init__(self, cert_info: Dict[str, Any]):
+        self._cert_info = self._decode_cert_data(cert_info)
+
+    @staticmethod
+    def from_url(url: str, timeout: int = 10) -> Optional['SSLCertificate']:
+        """
+        Create SSLCertificate instance from a URL.
+        
+        Args:
+            url (str): URL of the website.
+            timeout (int): Timeout for the connection (default: 10).
+        
+        Returns:
+            Optional[SSLCertificate]: SSLCertificate instance if successful, None otherwise.
+        """
+        try:
+            hostname = urlparse(url).netloc
+            if ':' in hostname:
+                hostname = hostname.split(':')[0]
+                
+            context = ssl.create_default_context()
+            with socket.create_connection((hostname, 443), timeout=timeout) as sock:
+                with context.wrap_socket(sock, server_hostname=hostname) as ssock:
+                    cert_binary = ssock.getpeercert(binary_form=True)
+                    x509 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_ASN1, cert_binary)
+                    
+                    cert_info = {
+                        "subject": dict(x509.get_subject().get_components()),
+                        "issuer": dict(x509.get_issuer().get_components()),
+                        "version": x509.get_version(),
+                        "serial_number": hex(x509.get_serial_number()),
+                        "not_before": x509.get_notBefore(),
+                        "not_after": x509.get_notAfter(),
+                        "fingerprint": x509.digest("sha256").hex(),
+                        "signature_algorithm": x509.get_signature_algorithm(),
+                        "raw_cert": base64.b64encode(cert_binary)
+                    }
+                    
+                    # Add extensions
+                    extensions = []
+                    for i in range(x509.get_extension_count()):
+                        ext = x509.get_extension(i)
+                        extensions.append({
+                            "name": ext.get_short_name(),
+                            "value": str(ext)
+                        })
+                    cert_info["extensions"] = extensions
+                    
+                    return SSLCertificate(cert_info)
+                    
+        except Exception as e:
+            return None
+
+    @staticmethod
+    def _decode_cert_data(data: Any) -> Any:
+        """Helper method to decode bytes in certificate data."""
+        if isinstance(data, bytes):
+            return data.decode('utf-8')
+        elif isinstance(data, dict):
+            return {
+                (k.decode('utf-8') if isinstance(k, bytes) else k): SSLCertificate._decode_cert_data(v)
+                for k, v in data.items()
+            }
+        elif isinstance(data, list):
+            return [SSLCertificate._decode_cert_data(item) for item in data]
+        return data
+
+    def to_json(self, filepath: Optional[str] = None) -> Optional[str]:
+        """
+        Export certificate as JSON.
+        
+        Args:
+            filepath (Optional[str]): Path to save the JSON file (default: None).
+        
+        Returns:
+            Optional[str]: JSON string if successful, None otherwise.
+        """
+        json_str = json.dumps(self._cert_info, indent=2, ensure_ascii=False)
+        if filepath:
+            Path(filepath).write_text(json_str, encoding='utf-8')
+            return None
+        return json_str
+
+    def to_pem(self, filepath: Optional[str] = None) -> Optional[str]:
+        """
+        Export certificate as PEM.
+        
+        Args:
+            filepath (Optional[str]): Path to save the PEM file (default: None).
+        
+        Returns:
+            Optional[str]: PEM string if successful, None otherwise.
+        """
+        try:
+            x509 = OpenSSL.crypto.load_certificate(
+                OpenSSL.crypto.FILETYPE_ASN1, 
+                base64.b64decode(self._cert_info['raw_cert'])
+            )
+            pem_data = OpenSSL.crypto.dump_certificate(
+                OpenSSL.crypto.FILETYPE_PEM, 
+                x509
+            ).decode('utf-8')
+            
+            if filepath:
+                Path(filepath).write_text(pem_data, encoding='utf-8')
+                return None
+            return pem_data
+        except Exception as e:
+            return None
+
+    def to_der(self, filepath: Optional[str] = None) -> Optional[bytes]:
+        """
+        Export certificate as DER.
+        
+        Args:
+            filepath (Optional[str]): Path to save the DER file (default: None).
+        
+        Returns:
+            Optional[bytes]: DER bytes if successful, None otherwise.
+        """
+        try:
+            der_data = base64.b64decode(self._cert_info['raw_cert'])
+            if filepath:
+                Path(filepath).write_bytes(der_data)
+                return None
+            return der_data
+        except Exception:
+            return None
+
+    @property
+    def issuer(self) -> Dict[str, str]:
+        """Get certificate issuer information."""
+        return self._cert_info.get('issuer', {})
+
+    @property
+    def subject(self) -> Dict[str, str]:
+        """Get certificate subject information."""
+        return self._cert_info.get('subject', {})
+
+    @property
+    def valid_from(self) -> str:
+        """Get certificate validity start date."""
+        return self._cert_info.get('not_before', '')
+
+    @property
+    def valid_until(self) -> str:
+        """Get certificate validity end date."""
+        return self._cert_info.get('not_after', '')
+
+    @property
+    def fingerprint(self) -> str:
+        """Get certificate fingerprint."""
+        return self._cert_info.get('fingerprint', '')

crawl4ai/user_agent_generator.py

@@ -0,0 +1,305 @@
+import random
+from typing import Optional, Literal, List, Dict, Tuple
+import re
+
+
+class UserAgentGenerator:
+    """
+    Generate random user agents with specified constraints.
+    
+    Attributes:
+        desktop_platforms (dict): A dictionary of possible desktop platforms and their corresponding user agent strings.
+        mobile_platforms (dict): A dictionary of possible mobile platforms and their corresponding user agent strings.
+        browser_combinations (dict): A dictionary of possible browser combinations and their corresponding user agent strings.
+        rendering_engines (dict): A dictionary of possible rendering engines and their corresponding user agent strings.
+        chrome_versions (list): A list of possible Chrome browser versions.
+        firefox_versions (list): A list of possible Firefox browser versions.
+        edge_versions (list): A list of possible Edge browser versions.
+        safari_versions (list): A list of possible Safari browser versions.
+        ios_versions (list): A list of possible iOS browser versions.
+        android_versions (list): A list of possible Android browser versions.
+        
+        Methods:
+            generate_user_agent(
+                platform: Literal["desktop", "mobile"] = "desktop",
+                browser: str = "chrome",
+                rendering_engine: str = "chrome_webkit",
+                chrome_version: Optional[str] = None,
+                firefox_version: Optional[str] = None,
+                edge_version: Optional[str] = None,
+                safari_version: Optional[str] = None,
+                ios_version: Optional[str] = None,
+                android_version: Optional[str] = None
+            ): Generates a random user agent string based on the specified parameters.    
+    """
+    def __init__(self):
+        # Previous platform definitions remain the same...
+        self.desktop_platforms = {
+            "windows": {
+                "10_64": "(Windows NT 10.0; Win64; x64)",
+                "10_32": "(Windows NT 10.0; WOW64)",
+            },
+            "macos": {
+                "intel": "(Macintosh; Intel Mac OS X 10_15_7)",
+                "newer": "(Macintosh; Intel Mac OS X 10.15; rv:109.0)",
+            },
+            "linux": {
+                "generic": "(X11; Linux x86_64)",
+                "ubuntu": "(X11; Ubuntu; Linux x86_64)",
+                "chrome_os": "(X11; CrOS x86_64 14541.0.0)",
+            }
+        }
+
+        self.mobile_platforms = {
+            "android": {
+                "samsung": "(Linux; Android 13; SM-S901B)",
+                "pixel": "(Linux; Android 12; Pixel 6)",
+                "oneplus": "(Linux; Android 13; OnePlus 9 Pro)",
+                "xiaomi": "(Linux; Android 12; M2102J20SG)",
+            },
+            "ios": {
+                "iphone": "(iPhone; CPU iPhone OS 16_5 like Mac OS X)",
+                "ipad": "(iPad; CPU OS 16_5 like Mac OS X)",
+            }
+        }
+
+        # Browser Combinations
+        self.browser_combinations = {
+            1: [
+                ["chrome"],
+                ["firefox"],
+                ["safari"],
+                ["edge"]
+            ],
+            2: [
+                ["gecko", "firefox"],
+                ["chrome", "safari"],
+                ["webkit", "safari"]
+            ],
+            3: [
+                ["chrome", "safari", "edge"],
+                ["webkit", "chrome", "safari"]
+            ]
+        }
+
+        # Rendering Engines with versions
+        self.rendering_engines = {
+            "chrome_webkit": "AppleWebKit/537.36",
+            "safari_webkit": "AppleWebKit/605.1.15",
+            "gecko": [  # Added Gecko versions
+                "Gecko/20100101",
+                "Gecko/20100101",  # Firefox usually uses this constant version
+                "Gecko/2010010",
+            ]
+        }
+
+        # Browser Versions
+        self.chrome_versions = [
+            "Chrome/119.0.6045.199",
+            "Chrome/118.0.5993.117",
+            "Chrome/117.0.5938.149",
+            "Chrome/116.0.5845.187",
+            "Chrome/115.0.5790.171",
+        ]
+
+        self.edge_versions = [
+            "Edg/119.0.2151.97",
+            "Edg/118.0.2088.76",
+            "Edg/117.0.2045.47",
+            "Edg/116.0.1938.81",
+            "Edg/115.0.1901.203",
+        ]
+
+        self.safari_versions = [
+            "Safari/537.36",  # For Chrome-based
+            "Safari/605.1.15",
+            "Safari/604.1",
+            "Safari/602.1",
+            "Safari/601.5.17",
+        ]
+
+        # Added Firefox versions
+        self.firefox_versions = [
+            "Firefox/119.0",
+            "Firefox/118.0.2",
+            "Firefox/117.0.1",
+            "Firefox/116.0",
+            "Firefox/115.0.3",
+            "Firefox/114.0.2",
+            "Firefox/113.0.1",
+            "Firefox/112.0",
+            "Firefox/111.0.1",
+            "Firefox/110.0",
+        ]
+
+    def get_browser_stack(self, num_browsers: int = 1) -> List[str]:
+        """
+        Get a valid combination of browser versions.
+        
+        How it works:
+        1. Check if the number of browsers is supported.
+        2. Randomly choose a combination of browsers.
+        3. Iterate through the combination and add browser versions.
+        4. Return the browser stack.
+        
+        Args:
+            num_browsers: Number of browser specifications (1-3)
+            
+        Returns:
+            List[str]: A list of browser versions.
+        """
+        if num_browsers not in self.browser_combinations:
+            raise ValueError(f"Unsupported number of browsers: {num_browsers}")
+        
+        combination = random.choice(self.browser_combinations[num_browsers])
+        browser_stack = []
+        
+        for browser in combination:
+            if browser == "chrome":
+                browser_stack.append(random.choice(self.chrome_versions))
+            elif browser == "firefox":
+                browser_stack.append(random.choice(self.firefox_versions))
+            elif browser == "safari":
+                browser_stack.append(random.choice(self.safari_versions))
+            elif browser == "edge":
+                browser_stack.append(random.choice(self.edge_versions))
+            elif browser == "gecko":
+                browser_stack.append(random.choice(self.rendering_engines["gecko"]))
+            elif browser == "webkit":
+                browser_stack.append(self.rendering_engines["chrome_webkit"])
+        
+        return browser_stack
+
+    def generate(self, 
+                device_type: Optional[Literal['desktop', 'mobile']] = None,
+                os_type: Optional[str] = None,
+                device_brand: Optional[str] = None,
+                browser_type: Optional[Literal['chrome', 'edge', 'safari', 'firefox']] = None,
+                num_browsers: int = 3) -> str:
+        """
+        Generate a random user agent with specified constraints.
+        
+        Args:
+            device_type: 'desktop' or 'mobile'
+            os_type: 'windows', 'macos', 'linux', 'android', 'ios'
+            device_brand: Specific device brand
+            browser_type: 'chrome', 'edge', 'safari', or 'firefox'
+            num_browsers: Number of browser specifications (1-3)
+        """
+        # Get platform string
+        platform = self.get_random_platform(device_type, os_type, device_brand)
+        
+        # Start with Mozilla
+        components = ["Mozilla/5.0", platform]
+        
+        # Add browser stack
+        browser_stack = self.get_browser_stack(num_browsers)
+        
+        # Add appropriate legacy token based on browser stack
+        if "Firefox" in str(browser_stack):
+            components.append(random.choice(self.rendering_engines["gecko"]))
+        elif "Chrome" in str(browser_stack) or "Safari" in str(browser_stack):
+            components.append(self.rendering_engines["chrome_webkit"])
+            components.append("(KHTML, like Gecko)")
+        
+        # Add browser versions
+        components.extend(browser_stack)
+        
+        return " ".join(components)
+
+    def generate_with_client_hints(self, **kwargs) -> Tuple[str, str]:
+        """Generate both user agent and matching client hints"""
+        user_agent = self.generate(**kwargs)
+        client_hints = self.generate_client_hints(user_agent)
+        return user_agent, client_hints
+
+    def get_random_platform(self, device_type, os_type, device_brand):
+        """Helper method to get random platform based on constraints"""
+        platforms = self.desktop_platforms if device_type == 'desktop' else \
+                   self.mobile_platforms if device_type == 'mobile' else \
+                   {**self.desktop_platforms, **self.mobile_platforms}
+        
+        if os_type:
+            for platform_group in [self.desktop_platforms, self.mobile_platforms]:
+                if os_type in platform_group:
+                    platforms = {os_type: platform_group[os_type]}
+                    break
+        
+        os_key = random.choice(list(platforms.keys()))
+        if device_brand and device_brand in platforms[os_key]:
+            return platforms[os_key][device_brand]
+        return random.choice(list(platforms[os_key].values()))
+
+    def parse_user_agent(self, user_agent: str) -> Dict[str, str]:
+        """Parse a user agent string to extract browser and version information"""
+        browsers = {
+            'chrome': r'Chrome/(\d+)',
+            'edge': r'Edg/(\d+)',
+            'safari': r'Version/(\d+)',
+            'firefox': r'Firefox/(\d+)'
+        }
+        
+        result = {}
+        for browser, pattern in browsers.items():
+            match = re.search(pattern, user_agent)
+            if match:
+                result[browser] = match.group(1)
+        
+        return result
+
+    def generate_client_hints(self, user_agent: str) -> str:
+        """Generate Sec-CH-UA header value based on user agent string"""
+        browsers = self.parse_user_agent(user_agent)
+        
+        # Client hints components
+        hints = []
+        
+        # Handle different browser combinations
+        if 'chrome' in browsers:
+            hints.append(f'"Chromium";v="{browsers["chrome"]}"')
+            hints.append('"Not_A Brand";v="8"')
+            
+            if 'edge' in browsers:
+                hints.append(f'"Microsoft Edge";v="{browsers["edge"]}"')
+            else:
+                hints.append(f'"Google Chrome";v="{browsers["chrome"]}"')
+                
+        elif 'firefox' in browsers:
+            # Firefox doesn't typically send Sec-CH-UA
+            return '""'
+            
+        elif 'safari' in browsers:
+            # Safari's format for client hints
+            hints.append(f'"Safari";v="{browsers["safari"]}"')
+            hints.append('"Not_A Brand";v="8"')
+        
+        return ', '.join(hints)
+
+# Example usage:
+if __name__ == "__main__":
+    generator = UserAgentGenerator()
+    print(generator.generate())
+    
+    print("\nSingle browser (Chrome):")
+    print(generator.generate(num_browsers=1, browser_type='chrome'))
+    
+    print("\nTwo browsers (Gecko/Firefox):")
+    print(generator.generate(num_browsers=2))
+    
+    print("\nThree browsers (Chrome/Safari/Edge):")
+    print(generator.generate(num_browsers=3))
+    
+    print("\nFirefox on Linux:")
+    print(generator.generate(
+        device_type='desktop',
+        os_type='linux',
+        browser_type='firefox',
+        num_browsers=2
+    ))
+    
+    print("\nChrome/Safari/Edge on Windows:")
+    print(generator.generate(
+        device_type='desktop',
+        os_type='windows',
+        num_browsers=3
+    ))