utils/cleaning.py

import subprocess
import sys
import re
import pandas as pd

try:
    import eyecite
except ImportError:
    subprocess.check_call([sys.executable, "-m", "pip", "install", "eyecite"])
finally:
    from eyecite import find, clean


# @title
def full_case(citation, text):
    text = text.replace(citation.matched_text(), "")
    if citation.metadata.year:
        pattern = r"\([^)]*{}\)".format(
            citation.metadata.year
        )  # Matches any word that ends with "year"
        text = re.sub(pattern, "", text)
    if citation.metadata.pin_cite:
        text = text.replace(citation.metadata.pin_cite, "")
    if citation.metadata.parenthetical:
        text = text.replace(f"({citation.metadata.parenthetical})", "")
    if citation.metadata.plaintiff:
        text = text.replace(
            f"{citation.metadata.plaintiff} v. {citation.metadata.defendant}", ""
        )
    publisher_date = " ".join(
        i for i in (citation.metadata.court, citation.metadata.year) if i
    )
    if publisher_date:
        text = text.replace(f"{publisher_date}", "")
    if citation.metadata.extra:
        text = text.replace(citation.metadata.extra, "")
    return text


def supra_case(citation, text):
    text = text.replace(citation.matched_text(), "")
    if citation.metadata.pin_cite:
        text = text.replace(citation.metadata.pin_cite, "")
    if citation.metadata.parenthetical:
        text = text.replace(f"({citation.metadata.parenthetical})", "")
    if citation.metadata.antecedent_guess:
        text = text.replace(citation.metadata.antecedent_guess, "")
    return text


def short_case(citation, text):
    text = text.replace(citation.matched_text(), "")
    if citation.metadata.parenthetical:
        text = text.replace(f"({citation.metadata.parenthetical})", "")
    if citation.metadata.year:
        pattern = r"\([^)]*{}\)".format(citation.metadata.year)
    if citation.metadata.antecedent_guess:
        text = text.replace(citation.metadata.antecedent_guess, "")
    return text


def id_case(citation, text):
    text = text.replace(citation.matched_text(), "")
    if citation.metadata.parenthetical:
        text = text.replace(f"({citation.metadata.parenthetical})", "")
    if citation.metadata.pin_cite:
        text = text.replace(citation.metadata.pin_cite, "")
    return text


def unknown_case(citation, text):
    text = text.replace(citation.matched_text(), "")
    if citation.metadata.parenthetical:
        text = text.replace(f"({citation.metadata.parenthetical})", "")
    return text


def full_law_case(citation, text):
    text = text.replace(citation.matched_text(), "")
    if citation.metadata.parenthetical:
        text = text.replace(f"({citation.metadata.parenthetical})", "")
    return text


def full_journal_case(citation, text):
    text = text.replace(citation.matched_text(), "")
    if citation.metadata.year:
        pattern = r"\([^)]*{}\)".format(
            citation.metadata.year
        )  # Matches any word that ends with "year"
        text = re.sub(pattern, "", text)
    if citation.metadata.pin_cite:
        text = text.replace(citation.metadata.pin_cite, "")
    if citation.metadata.parenthetical:
        text = text.replace(f"({citation.metadata.parenthetical})", "")
    return text


def all_commas(text: str) -> str:
    return re.sub(r"\,+", ",", text)


def all_dots(text: str) -> str:
    return re.sub(r"\.+", ".", text)


functions_dict = {
    "FullCaseCitation": full_case,
    "SupraCitation": supra_case,
    "ShortCaseCitation": short_case,
    "IdCitation": id_case,
    "UnknownCitation": unknown_case,
    "FullLawCitation": full_law_case,
    "FullJournalCitation": full_journal_case,
}


# @title
def remove_citations(input_text):
    # clean text
    plain_text = clean.clean_text(
        input_text, ["html", "inline_whitespace", "underscores"]
    )
    # remove citations
    found_citations = find.get_citations(plain_text)
    for citation in found_citations:
        plain_text = functions_dict[citation.__class__.__name__](citation, plain_text)
    # clean text
    plain_text = clean.clean_text(
        plain_text,
        ["inline_whitespace", "underscores", "all_whitespace", all_commas, all_dots],
    )
    plain_text = clean.clean_text(plain_text, ["inline_whitespace", "all_whitespace"])
    pattern = r"\*?\d*\s*I+\n"
    plain_text = re.sub(pattern, "", plain_text)
    pattern = r"\s[,.]"
    plain_text = re.sub(pattern, "", plain_text)
    return plain_text


def split_text(text):
    words = text.split()
    chunks = []
    for i in range(0, len(words), 420):
        chunks.append(" ".join(words[i : i + 430]))
    return chunks


# @title
def chunk_text_to_paragraphs(text):
    paragraphs = text.split("\n")  # Split by empty line

    # Remove leading and trailing whitespace from each paragraph
    paragraphs = [p.strip() for p in paragraphs]

    return paragraphs


# @title
def split_data(data, id2label, label2id):
    data_dict = {
        "author_name": [],
        "label": [],
        "category": [],
        "case_name": [],
        "url": [],
        "text": [],
    }
    opinions_split = pd.DataFrame(data_dict)
    opinions_split["label"] = opinions_split["label"].astype(int)
    for index, row in data.iterrows():
        # chunks = chunk_text_to_paragraphs(row['text'])
        chunks = split_text(row["clean_text"])
        for chunk in chunks:
            if len(chunk) < 1000:
                continue
            tmp = pd.DataFrame(
                {
                    "author_name": row["author_name"],
                    "label": [label2id[row["author_name"]]],
                    "category": row["category"],
                    "case_name": row["case_name"],
                    "url": [row["absolute_url"]],
                    "text": [chunk],
                }
            )
            opinions_split = pd.concat([opinions_split, tmp])
    return opinions_split


def chunk_data(data):
    data_dict = {"text": []}
    opinions_split = pd.DataFrame(data_dict)
    chunks = split_text(data)
    for chunk in chunks:
        # if len(chunk) < 1000:
        #     continue
        tmp = pd.DataFrame({"label": [200], "text": [chunk]})
        opinions_split = pd.concat([opinions_split, tmp])
    return opinions_split