web_scrapping_engine.py

import os
from scrappers.linkedin import get_linkedin_profile
from scrappers.company_website import full_company_website_exploration

def run_web_scrapping_engine(company_name = "sunday natural", folder_path = "data_dumpster", output_in_code = False):
    path = f"{folder_path}/{company_name.replace(' ','-')}"
    os.makedirs(path, exist_ok=True)
    os.makedirs(f"{path}/pdf/", exist_ok=True)
    print("Starting Linkedin gathering...")
    get_linkedin_profile(company_name, folder_path=path)
    print("Linkedin Done!\n========> Starting now company website gathering...")
    content = full_company_website_exploration(company_name, folder_path=path)
    if output_in_code:
        return content