add files

.gitignore

@@ -0,0 +1,4 @@
+__pycache__
+flagged_data
+.gitattributes
+.gradio

README copy.md

@@ -0,0 +1,12 @@
+---
+title: Hyw En Demo v2
+emoji: ⚡
+colorFrom: blue
+colorTo: purple
+sdk: gradio
+sdk_version: 4.43.0
+app_file: app.py
+pinned: false
+license: cc-by-4.0
+---
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

README.md

@@ -1,12 +1,12 @@
 ---
 title: Xcl En Demo
-emoji: 🐨
-colorFrom: pink
-colorTo: red
+emoji: 📖
+colorFrom: red
+colorTo: purple
 sdk: gradio
-sdk_version: 5.9.1
+sdk_version: 4.43.0
 app_file: app.py
 pinned: false
+license: cc-by-4.0
 ---
-
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,231 @@
+# coding: utf-8
+
+import gradio as gr
+import os
+import json
+from pathlib import Path
+from uuid import uuid4
+from datetime import datetime
+from huggingface_hub import CommitScheduler
+from translation import Translator, LANGUAGES
+import re
+
+
+LANGUAGES_LIST = list(LANGUAGES.keys())
+HF_TOKEN = os.environ.get('HF_TOKEN')
+
+JSON_DATASET_DIR = Path("flagged_data")
+JSON_DATASET_DIR.mkdir(exist_ok=True, parents=True)
+JSON_DATASET_PATH = JSON_DATASET_DIR / f"dataset-session-{uuid4()}.json"
+
+
+
+
+def translate_wrapper(text, src_lang, tgt_lang, by_sentence=True, clean=True, num_beams=4):
+    if text in ["", None, []]:
+        return "Մուտքագրումը պարապ է։ | Input is empty."
+
+    if src_lang in ["", None, []] or tgt_lang in ["", None, []]:
+        return "Ընտրեցէք թարգմանութեան կողմերը | Please select source and target languages"
+
+    if src_lang == tgt_lang:
+        return "Ընտրուած լեզուները նոյնն են։ | Source and target languages are identical."
+
+    src_lang = LANGUAGES.get(src_lang)
+    tgt_lang = LANGUAGES.get(tgt_lang)
+
+    result = translator.translate(text, src_lang, tgt_lang, by_sentence=by_sentence, clean=clean, num_beams=num_beams)
+    return result
+
+
+# hf_writer = gr.HuggingFaceDatasetSaver(HF_TOKEN, "AriNubar/hyw-en-crowd-source")
+
+scheduler = CommitScheduler(
+    repo_id="AriNubar/xcl-en-crowdsource",
+    repo_type="dataset",
+    folder_path=JSON_DATASET_DIR,
+    path_in_repo="collected_data",
+    token=HF_TOKEN,
+    every=30 # every 30 minute
+)
+
+TQS = {
+    "😊 Լաւ | Good": "good",
+    "😐 Միջակ | Average": "average",
+    "☹️ Վատ | Bad": "bad"
+}
+
+def save_json(src_lang, tgt_lang, input_text, output_text, by_sentence, clean, num_beams, translation_quality):
+    if any([src_lang in ["", None, []], tgt_lang in ["", None, []], input_text in ["", None, []], output_text in ["", None, []]]):
+        gr.Warning("Տուեալին մէկ մասը պարապ է։ Ձեր գնահատութիւնը չպահուեցաւ։ | Some part of the data is missing. Your feedback has not been saved.")
+        return 
+    
+    src_lang = LANGUAGES.get(src_lang)
+    tgt_lang = LANGUAGES.get(tgt_lang)
+    translation_quality = TQS.get(translation_quality)
+    
+    # print(src_lang, tgt_lang, input_text, output_text, by_sentence, clean, num_beams, translation_quality)
+    # print(type(src_lang), type(tgt_lang), type(input_text), type(output_text), type(by_sentence), type(clean), type(num_beams), type(translation_quality))
+    data = {
+        "src_lang": src_lang,
+        "tgt_lang": tgt_lang,
+        "original": input_text,
+        "translation": output_text,
+        "by_sentence": by_sentence,
+        "clean": clean,
+        "num_beams": num_beams,
+        "translation_quality": translation_quality,
+        "timestamp": datetime.now().isoformat()
+    }
+    with scheduler.lock:
+        with open(JSON_DATASET_PATH, "a", encoding="utf8") as f:
+            f.write(json.dumps(data, ensure_ascii=False) + "\n")
+
+    gr.Info("Ձեր գնահանութիւնը պահուեցաւ։ Շատ շնորհակալութի՛ւն։ | Your feedback has been saved. Thank you.")
+
+
+def switch_languages(src, tgt, input_text, output_text):
+    new_src = tgt
+    new_tgt = src
+    new_input = output_text if output_text else input_text
+    return [new_src, new_tgt, new_input, None]
+
+
+def detect_language(text):
+    """Detect language based on script ratio"""
+    armenian_pattern = r'[\u0531-\u0587\u0589\u058A\u058F]'
+    non_armenian_pattern = r'[a-zA-Z]'
+    
+    armenian_chars = len(re.findall(armenian_pattern, text))
+    non_armenian_chars = len(re.findall(non_armenian_pattern, text))
+    
+    if armenian_chars > non_armenian_chars:
+        return "Գրաբար Հայոց | Classical Armenian", "Անգլերէն | English"
+    elif non_armenian_chars > 0:
+        return "Անգլերէն | English", "Գրաբար Հայոց | Classical Armenian"
+    return [gr.update(), gr.update()]  # No clear dominance, reset dropdowns
+
+def update_languages(text):
+    if not text:
+        return [gr.update(), gr.update()]
+    src, tgt = detect_language(text)
+    return [gr.update(value=src), gr.update(value=tgt)]
+
+
+theme = gr.themes.Default().set(
+    block_info_text_size="*text_xxs" # for info text
+)
+
+with gr.Blocks(title="Գրաբար-Անգլերէն Մեքենական Թարգմանիչ | Classical Armenian-English Machine Translation", 
+               theme=theme,
+               ) as demo:
+    
+    gr.HTML("""
+        <h2 style='margin-bottom: 5px'>Գրաբար-Անգլերէն Մեքենական Թարգմանիչ | Classical Armenian-English Machine Translation</h2>
+        <h3>Տարբերակ | Version: 1.0</h3>
+        <h3 style='margin-bottom: 5px'>Ստեղծող՝ | Created By: <a href='https://www.arinubar.com' target='_blank'>Ari Nubar Boyacıoğlu</a></h3>
+        <p style="font-size: 0.7rem">Եթէ այս գործիքը կարողացաւ ձեզ օգտակար հանդիսանալ, բարելաւելու համար հաճեցէք սուրճի մը փոխարժէքը նուիրել․ | If this tool has proven useful to you, please consider making a donation. <a href='https://www.paypal.com/donate/?hosted_button_id=RRBCV3GQJ7D8N' target='_blank'>PayPal</a> | <a href='https://buymeacoffee.com/arinubar' target='_blank'>Buy Me a Coffee</a></p>
+    """)
+
+    with gr.Accordion("Թարգմանիչի Մասին | Information about the Translator", open=False):
+        gr.HTML("""
+            <p>Հոս կը ցուցադրուի առաջին գրաբար-անգլերէն մեքենական թարգմանիչը, որ կարուցուած է Մեթայի (Ֆեյսպուքի) 'No Language Left Behind' տիպարի հիման վրայ։ Թարգմանութեան տիպարը կ'աշխատի CPU-ի մը մէջ, ուրեմն նախադասութեան մը թարգմանութիւնը կրնայ տեւել մօտաւորապէս <strong>40-60 երկվայրկեան</strong>։ Ձեր գնահատութիւնները եւ քննադատութիւնները շատ կարեւոր են տիպարի թարգմանութեան որակը բարելաւելու համար։</p>
+            <p>Դուք թարգմանութեան որակին մասին ձեր գնահատութիւնը կրնաք տալ երեք գնահատութեան կոճակներէ մէկուն սեղմելով։ Լեզուի, մուտքի եւ ելքի գրութիւններու, յարաչափերու եւ ձեր գնահատութեան մասին տուեալները պիտի պահուին։ Գնահատութիւնը պարտաւոր չէ։</p>
+            <hr style='margin-top: 5px; margin-bottom: 5px'>
+            <p>This is the demo of the first Classical Armenian-English neural machine translation system which is based on Meta's 'No Language Left Behind' model. The model runs on a CPU, so it might take approximately <strong>40-60 seconds</strong> to translate a single sentence. Your feedback and comments are very important for us to improve the quality of the translation.</p>
+            <p>You can give your feedback about the quality of the translation by clicking one of the three feedback buttons. Information about source, target languages, input and output texts, parameters and your feedback about quality will be saved. It is not mandatory to give feedback.</p>
+        """)
+    
+    with gr.Row():
+        with gr.Column():
+            text = gr.Textbox(
+                lines=5, 
+                label="Մուտքագրում | Input Text",
+                every=1.5  # Trigger event 1.5 seconds after last keystroke
+            )
+
+            with gr.Row():
+                src_lang = gr.Dropdown(LANGUAGES_LIST, type="value", label="Թարգմանէ Այս Լեզուէ | Source Language")
+                tgt_lang = gr.Dropdown(LANGUAGES_LIST, type="value", label="Թարգմանէ Այս Լեզուի | Target Language")
+
+            with gr.Row():
+                switch_btn = gr.Button("🔄 Լեզուները Փոխէ | Switch Languages")
+
+
+            def switch_languages(src, tgt, input_text, output_text):
+                # Swap languages
+                new_src = tgt
+                new_tgt = src
+                # Move output to input if exists and clear output
+                new_input = output_text if output_text else input_text
+                return [new_src, new_tgt, new_input, None]
+            
+            text.change(fn=update_languages, 
+                       inputs=[text], 
+                       outputs=[src_lang, tgt_lang])
+
+        with gr.Column():
+            translated = gr.Textbox(lines=5, label="Ելքագրում | Output Text", interactive=False)
+            translate_btn = gr.Button(value="Թարգմանէ | Translate", variant="primary")
+            with gr.Row():
+                with gr.Column():
+                    gr.Markdown("""
+                    ### Թարգմանութեան Որակ | Translation Quality
+                    """)                
+                    flag_good_btn = gr.Button(value="😊 Լաւ | Good", size="sm")
+                    flag_average_btn = gr.Button(value="😐 Միջակ | Average", size="sm")
+                    flag_bad_btn = gr.Button(value="☹️ Վատ | Bad", size="sm")
+
+    with gr.Row():
+        gr.Markdown("""
+            ## Յարաչափեր | Parameters
+        """
+        )
+        by_sentence = gr.Checkbox(label="Նախադասութիւններու Բաժնէ | Split into Sentences", value=True, info="Տուփը նշանագրեցէք եթէ կ'ուզէք ձեր մուտքագրումը թարգմանուի նախադասութիւն առ նախադասութիւն։ Այս կերպով թարգմանուած նախադասութիւններուն որակը ընդհանրապէս աւելի լաւ կ'ըլլան։ | Check this box if you want to split your input text into sentences. This way the quality of the translation will be better.")
+        clean = gr.Checkbox(label="Մշակէ | Preprocess", value=True, info="Տուփը նշանագրեցէք եթէ կ'ուզէք ձեր մուտքագրումը կանոնաւորուի ծրագրի կողմէ թարգմանութենէ առաջ։ Կանոնաւորումը թարգմանութեան որակի բարելաւման համար օգտակար է։ | Check this box if you want to preprocess your input text before translation. This way the quality of the translation will be better.")
+        num_beams = gr.Dropdown([1, 2, 3, 4, 5], type="value", label="Որոնման Շողեր | Number of Beams", value=4, info="Աւելի բարձր թիւը ընդհանրապէս կը պատճառէ աւելի բարձր որակի, բայց նոյնիսկ երկարատեւ թարգմանութեան։ | Higher beam size will result in better quality translation, but also longer translation time.")
+
+    switch_btn.click(switch_languages, inputs=[src_lang, tgt_lang, text, translated], outputs=[src_lang, tgt_lang, text, translated])
+    translate_btn.click(translate_wrapper, inputs=[text, src_lang, tgt_lang, by_sentence, clean, num_beams], outputs=translated)
+    
+    # hf_writer.setup([src_lang, tgt_lang, text, translated, by_sentence, clean, num_beams, flag_bad_btn], "flagged_data_points")
+    
+    flag_good_btn.click(save_json, inputs=[src_lang, tgt_lang, text, translated, by_sentence, clean, num_beams, flag_good_btn], outputs=None)
+    flag_average_btn.click(save_json, inputs=[src_lang, tgt_lang, text, translated, by_sentence, clean, num_beams, flag_average_btn], outputs=None)
+    flag_bad_btn.click(save_json, inputs=[src_lang, tgt_lang, text, translated, by_sentence, clean, num_beams, flag_bad_btn], outputs=None)
+
+    visitor_badge_html = """
+        <a href="https://visitorbadge.io/status?path=https%3A%2F%2Farinubar-hyw-en-demo.hf.space%2F">
+            <img src="https://api.visitorbadge.io/api/visitors?path=https%3A%2F%2Farinubar-hyw-en-demo.hf.space%2F&label=%D4%B1%D5%B5%D6%81%D5%A5%D5%AC%D5%B8%D6%82%D5%B6%D5%A5%D6%80%20%7C%20Visitors&countColor=%23f97316&style=flat" />
+        </a>
+    """
+    
+    gr.HTML(visitor_badge_html)
+
+
+
+    sponsors_html = """
+        <div style="display: flex; justify-content: center; align-items: center; margin-bottom: 5px;">
+            <h3>Աջակցութեամբ՝ | Supported By: </h3>
+        </div>
+        <div style="display: flex; justify-content: center; align-items: center; background-color: #0000007a; border-radius: 20px;">
+            <a href="#">
+                <img src="/file=./img/mkhitaryan-varjaran.png" alt="Pangalti Mkhitaryan School" style="padding: 10px; margin: 20px; width: 150px;" />
+            </a>
+
+            <a href="https://gulbenkian.pt/armenian-communities/">
+                <img src="/file=./img/gulbenkian.png" alt="Calouste Gulbenkian Foundation - Armenian Communities" style="padding: 10px; margin: 20px;" />
+            </a>
+
+            <a href="http://www.teaov.org/">
+                <img src="/file=./img/teaov.png" alt="Turkish-Armenian Minority Schools Teachers Foundation" style="padding: 10px; margin: 20px; width: 200px; padding-right:35px;" />
+            </a>
+        </div>
+
+    """
+    gr.HTML(sponsors_html)
+
+
+if __name__ == "__main__":
+    translator = Translator()
+    demo.launch(favicon_path="img/translate.png", share=True, allowed_paths=["./img"])

pysbd/__init__.py

@@ -0,0 +1,2 @@
+from .segmenter import Segmenter
+from .about import __version__

pysbd/abbreviation_replacer.py

@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*-
+import re
+from pysbd.utils import Text
+
+
+def replace_pre_number_abbr(txt, abbr):
+    # prepend a space to avoid needing another regex for start of string
+    txt = " " + txt
+    txt = re.sub(r"(?<=\s{abbr})\.(?=(\s\d|\s+\())".format(abbr=abbr.strip()), "∯", txt)
+    # remove the prepended space
+    txt = txt[1:]
+    return txt
+
+
+def replace_prepositive_abbr(txt, abbr):
+    # prepend a space to avoid needing another regex for start of string
+    txt = " " + txt
+    txt = re.sub(r"(?<=\s{abbr})\.(?=(\s|:\d+))".format(abbr=abbr.strip()), "∯", txt)
+    # remove the prepended space
+    txt = txt[1:]
+    return txt
+
+
+class AbbreviationReplacer(object):
+    def __init__(self, text, lang):
+        self.text = text
+        self.lang = lang
+
+    def replace(self):
+        self.text = Text(self.text).apply(
+            self.lang.PossessiveAbbreviationRule,
+            self.lang.KommanditgesellschaftRule,
+            *self.lang.SingleLetterAbbreviationRules.All
+        )
+        abbr_handled_text = ""
+        for line in self.text.splitlines(True):
+            abbr_handled_text += self.search_for_abbreviations_in_string(line)
+        self.text = abbr_handled_text
+        self.replace_multi_period_abbreviations()
+        self.text = Text(self.text).apply(*self.lang.AmPmRules.All)
+        self.text = self.replace_abbreviation_as_sentence_boundary()
+        return self.text
+
+    def replace_abbreviation_as_sentence_boundary(self):
+        sent_starters = "|".join((r"(?=\s{}\s)".format(word) for word in self.SENTENCE_STARTERS))
+        regex = r"(U∯S|U\.S|U∯K|E∯U|E\.U|U∯S∯A|U\.S\.A|I|i.v|I.V)∯({})".format(sent_starters)
+        self.text = re.sub(regex, '\\1.', self.text)
+        return self.text
+
+    def replace_multi_period_abbreviations(self):
+        def mpa_replace(match):
+            match = match.group()
+            match = re.sub(re.escape(r"."), "∯", match)
+            return match
+
+        self.text = re.sub(
+            self.lang.MULTI_PERIOD_ABBREVIATION_REGEX,
+            mpa_replace,
+            self.text,
+            flags=re.IGNORECASE
+        )
+
+    def replace_period_of_abbr(self, txt, abbr):
+        # prepend a space to avoid needing another regex for start of string
+        txt = " " + txt
+        txt = re.sub(
+            r"(?<=\s{abbr})\.(?=((\.|\:|-|\?|,)|(\s([a-z]|I\s|I'm|I'll|\d|\())))".format(
+                abbr=re.escape(abbr.strip())
+            ),
+            "∯",
+            txt,
+        )
+        # remove the prepended space
+        txt = txt[1:]
+        return txt
+
+
+    def search_for_abbreviations_in_string(self, text):
+        lowered = text.lower()
+        for abbr in self.lang.Abbreviation.ABBREVIATIONS:
+            stripped = abbr.strip()
+            if stripped not in lowered:
+                continue
+            abbrev_match = re.findall(
+                r"(?:^|\s|\r|\n){}".format(stripped), text, flags=re.IGNORECASE
+            )
+            if not abbrev_match:
+                continue
+            next_word_start = r"(?<={" + str(re.escape(stripped)) + "} ).{1}"
+            char_array = re.findall(next_word_start, text)
+            for ind, match in enumerate(abbrev_match):
+                text = self.scan_for_replacements(
+                    text, match, ind, char_array
+                )
+        return text
+
+    def scan_for_replacements(self, txt, am, ind, char_array):
+        try:
+            char = char_array[ind]
+        except IndexError:
+            char = ""
+        prepositive = self.lang.Abbreviation.PREPOSITIVE_ABBREVIATIONS
+        number_abbr = self.lang.Abbreviation.NUMBER_ABBREVIATIONS
+        upper = str(char).isupper()
+        if not upper or am.strip().lower() in prepositive:
+            if am.strip().lower() in prepositive:
+                txt = replace_prepositive_abbr(txt, am)
+            elif am.strip().lower() in number_abbr:
+                txt = replace_pre_number_abbr(txt, am)
+            else:
+                txt = self.replace_period_of_abbr(txt, am)
+        return txt

pysbd/about.py

@@ -0,0 +1,10 @@
+# inspired from:
+# https://python-packaging-user-guide.readthedocs.org/en/latest/single_source_version/
+
+__title__ = "pysbd"
+__version__ = "0.3.4"
+__summary__ = "pysbd (Python Sentence Boundary Disambiguation) is a rule-based sentence boundary detection that works out-of-the-box across many languages."
+__uri__ = "http://nipunsadvilkar.github.io/"
+__author__ = "Nipun Sadvilkar"
+__email__ = "[email protected]"
+__license__ = "MIT"

pysbd/between_punctuation.py

@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*-
+import re
+from functools import partial
+from pysbd.punctuation_replacer import replace_punctuation
+
+
+class BetweenPunctuation(object):
+    # Rubular: http://rubular.com/r/2YFrKWQUYi
+    BETWEEN_SINGLE_QUOTES_REGEX = r"(?<=\s)'(?:[^']|'[a-zA-Z])*'"
+
+    BETWEEN_SINGLE_QUOTE_SLANTED_REGEX = r"(?<=\s)‘(?:[^’]|’[a-zA-Z])*’"
+
+    # Rubular: http://rubular.com/r/3Pw1QlXOjd
+    BETWEEN_DOUBLE_QUOTES_REGEX = r'"(?>[^"\\]+|\\{2}|\\.)*"'
+
+    # https://regex101.com/r/r6I1bW/1
+    # https://stackoverflow.com/questions/13577372/do-python-regular-expressions-have-an-equivalent-to-rubys-atomic-grouping?noredirect=1&lq=1
+    BETWEEN_DOUBLE_QUOTES_REGEX_2 = r'"(?=(?P<tmp>[^\"\\]+|\\{2}|\\.)*)(?P=tmp)"'
+
+    # Rubular: http://rubular.com/r/x6s4PZK8jc
+    BETWEEN_QUOTE_ARROW_REGEX = r'«(?>[^»\\]+|\\{2}|\\.)*»'
+
+    BETWEEN_QUOTE_ARROW_REGEX_2 = r"\«(?=(?P<tmp>[^»\\]+|\\{2}|\\.)*)(?P=tmp)\»"
+
+    # Rubular: http://rubular.com/r/JbAIpKdlSq
+    BETWEEN_QUOTE_SLANTED_REGEX = r"“(?>[^”\\]+|\\{2}|\\.)*”"
+    BETWEEN_QUOTE_SLANTED_REGEX_2 = r"\“(?=(?P<tmp>[^”\\]+|\\{2}|\\.)*)(?P=tmp)\”"
+
+    # Rubular: http://rubular.com/r/WX4AvnZvlX
+    BETWEEN_SQUARE_BRACKETS_REGEX = r"\[(?>[^\]\\]+|\\{2}|\\.)*\]"
+
+    BETWEEN_SQUARE_BRACKETS_REGEX_2 = r'\[(?=(?P<tmp>[^\]\\]+|\\{2}|\\.)*)(?P=tmp)\]'
+
+    # Rubular: http://rubular.com/r/6tTityPflI
+    BETWEEN_PARENS_REGEX = r"\((?>[^\(\)\\]+|\\{2}|\\.)*\)"
+
+    BETWEEN_PARENS_REGEX_2 = r"\((?=(?P<tmp>[^\(\)\\]+|\\{2}|\\.)*)(?P=tmp)\)"
+
+    # Rubular: http://rubular.com/r/mXf8cW025o
+    WORD_WITH_LEADING_APOSTROPHE = r"(?<=\s)'(?:[^']|'[a-zA-Z])*'\S"
+
+    # Rubular: http://rubular.com/r/jTtDKfjxzr
+    BETWEEN_EM_DASHES_REGEX = r"\-\-(?>[^\-\-])*\-\-"
+
+    BETWEEN_EM_DASHES_REGEX_2 = r"--(?=(?P<tmp>[^--]*))(?P=tmp)--"
+
+    def __init__(self, text):
+        self.text = text
+
+    def replace(self):
+        return self.sub_punctuation_between_quotes_and_parens(self.text)
+
+    def sub_punctuation_between_quotes_and_parens(self, txt):
+        txt = self.sub_punctuation_between_single_quotes(txt)
+        txt = self.sub_punctuation_between_single_quote_slanted(txt)
+        txt = self.sub_punctuation_between_double_quotes(txt)
+        txt = self.sub_punctuation_between_square_brackets(txt)
+        txt = self.sub_punctuation_between_parens(txt)
+        txt = self.sub_punctuation_between_quotes_arrow(txt)
+        txt = self.sub_punctuation_between_em_dashes(txt)
+        txt = self.sub_punctuation_between_quotes_slanted(txt)
+        return txt
+
+    def sub_punctuation_between_parens(self, txt):
+        return re.sub(self.BETWEEN_PARENS_REGEX_2, replace_punctuation, txt)
+
+    def sub_punctuation_between_square_brackets(self, txt):
+        return re.sub(self.BETWEEN_SQUARE_BRACKETS_REGEX_2, replace_punctuation,
+                      txt)
+
+    def sub_punctuation_between_single_quotes(self, txt):
+        if re.search(self.WORD_WITH_LEADING_APOSTROPHE, txt) and \
+                (not re.search(r"'\s", txt)):
+            return txt
+        return re.sub(self.BETWEEN_SINGLE_QUOTES_REGEX,
+                      partial(replace_punctuation, match_type='single'), txt)
+
+    def sub_punctuation_between_single_quote_slanted(self, txt):
+        return re.sub(self.BETWEEN_SINGLE_QUOTE_SLANTED_REGEX,
+                      replace_punctuation, txt)
+
+    def sub_punctuation_between_double_quotes(self, txt):
+        return re.sub(self.BETWEEN_DOUBLE_QUOTES_REGEX_2, replace_punctuation,
+                      txt)
+
+    def sub_punctuation_between_quotes_arrow(self, txt):
+        return re.sub(self.BETWEEN_QUOTE_ARROW_REGEX_2, replace_punctuation, txt)
+
+    def sub_punctuation_between_em_dashes(self, txt):
+        return re.sub(self.BETWEEN_EM_DASHES_REGEX_2, replace_punctuation, txt)
+
+    def sub_punctuation_between_quotes_slanted(self, txt):
+        return re.sub(self.BETWEEN_QUOTE_SLANTED_REGEX_2, replace_punctuation,
+                      txt)

pysbd/clean/rules.py

@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+from pysbd.utils import Rule
+
+
+class CleanRules(object):
+
+    # NOTE: Caution: Might require \\ for special characters
+    # if regex is defined with r'' then dont
+    # add extra \\ for special characters
+    # Rubular: http://rubular.com/r/V57WnM9Zut
+    NewLineInMiddleOfWordRule = Rule(r'\n(?=[a-zA-Z]{1,2}\n)', '')
+
+    # Rubular: http://rubular.com/r/dMxp5MixFS
+    DoubleNewLineWithSpaceRule = Rule(r'\n \n', "\r")
+
+    # Rubular: http://rubular.com/r/H6HOJeA8bq
+    DoubleNewLineRule = Rule(r'\n\n', "\r")
+
+    # Rubular: http://rubular.com/r/FseyMiiYFT
+    NewLineFollowedByPeriodRule = Rule(r'\n(?=\.(\s|\n))', '')
+
+    ReplaceNewlineWithCarriageReturnRule = Rule(r'\n', "\r")
+
+    EscapedNewLineRule = Rule(r'\\n', "\n")
+
+    EscapedCarriageReturnRule = Rule(r'\\r', "\r")
+
+    TypoEscapedNewLineRule = Rule(r'\\\ n', "\n")
+
+    TypoEscapedCarriageReturnRule = Rule(r'\\\ r', "\r")
+
+    # Rubular: http://rubular.com/r/bAJrhyLNeZ
+    InlineFormattingRule = Rule(r'{b\^&gt;\d*&lt;b\^}|{b\^>\d*<b\^}', '')
+
+    # Rubular: http://rubular.com/r/8mc1ArOIGy
+    TableOfContentsRule = Rule(r'\.{4,}\s*\d+-*\d*', "\r")
+
+    # Rubular: http://rubular.com/r/DwNSuZrNtk
+    ConsecutivePeriodsRule = Rule(r'\.{5,}', ' ')
+
+    # Rubular: http://rubular.com/r/IQ4TPfsbd8
+    ConsecutiveForwardSlashRule = Rule(r'\/{3}', '')
+
+    # Rubular: http://rubular.com/r/6dt98uI76u
+    NO_SPACE_BETWEEN_SENTENCES_REGEX = r'(?<=[a-z])\.(?=[A-Z])'
+    # NO_SPACE_BETWEEN_SENTENCES_REGEX = r'[a-z]\.[A-Z]'
+    NoSpaceBetweenSentencesRule = Rule(NO_SPACE_BETWEEN_SENTENCES_REGEX, '. ')
+
+    # Rubular: http://rubular.com/r/l6KN6rH5XE
+    NO_SPACE_BETWEEN_SENTENCES_DIGIT_REGEX = r'(?<=\d)\.(?=[A-Z])'
+    NoSpaceBetweenSentencesDigitRule = Rule(NO_SPACE_BETWEEN_SENTENCES_DIGIT_REGEX, '. ')
+
+    URL_EMAIL_KEYWORDS = ['@', 'http', '.com', 'net', 'www', '//']
+
+    # Rubular: http://rubular.com/r/3GiRiP2IbD
+    NEWLINE_IN_MIDDLE_OF_SENTENCE_REGEX = r'(?<=\s)\n(?=([a-z]|\())'
+
+    # Rubular: http://rubular.com/r/Gn18aAnLdZ
+    NewLineFollowedByBulletRule = Rule(r"\n(?=•')", "\r")
+
+    QuotationsFirstRule = Rule(r"''", '"')
+    QuotationsSecondRule = Rule(r'``', '"')
+
+
+class HTML(object):
+    # Rubular: http://rubular.com/r/9d0OVOEJWj
+    HTMLTagRule = Rule(r"<\/?\w+((\s+\w+(\s*=\s*(?:\".*?\"|'.*?'|[\^'\">\s]+))?)+\s*|\s*)\/?>", '')
+
+    # Rubular: http://rubular.com/r/XZVqMPJhea
+    EscapedHTMLTagRule = Rule(r'&lt;\/?[^gt;]*gt;', '')
+
+    All = [HTMLTagRule, EscapedHTMLTagRule]
+
+
+class PDF(object):
+    # Rubular: http://rubular.com/r/UZAVcwqck8
+    NewLineInMiddleOfSentenceRule = Rule(r'(?<=[^\n]\s)\n(?=\S)', '')
+
+    # Rubular: http://rubular.com/r/eaNwGavmdo
+    NewLineInMiddleOfSentenceNoSpacesRule = Rule(r"\n(?=[a-z])", ' ')

pysbd/cleaner.py

@@ -0,0 +1,111 @@
+# -*- coding: utf-8 -*-
+import re
+from pysbd.utils import Text
+from pysbd.clean.rules import PDF, HTML, CleanRules as cr
+
+
+class Cleaner(object):
+
+    def __init__(self, text, lang, doc_type=None):
+        self.text = text
+        self.lang = lang
+        self.doc_type = doc_type
+
+    def clean(self):
+        if not self.text:
+            return self.text
+        self.remove_all_newlines()
+        self.replace_double_newlines()
+        self.replace_newlines()
+        self.replace_escaped_newlines()
+        self.text = Text(self.text).apply(*HTML.All)
+        self.replace_punctuation_in_brackets()
+        self.text = Text(self.text).apply(cr.InlineFormattingRule)
+        self.clean_quotations()
+        self.clean_table_of_contents()
+        self.check_for_no_space_in_between_sentences()
+        self.clean_consecutive_characters()
+        return self.text
+
+    def remove_all_newlines(self):
+        self.remove_newline_in_middle_of_sentence()
+        self.remove_newline_in_middle_of_word()
+
+    def remove_newline_in_middle_of_sentence(self):
+        def replace_w_blank(match):
+            match = match.group()
+            sub = re.sub(cr.NEWLINE_IN_MIDDLE_OF_SENTENCE_REGEX, '', match)
+            return sub
+        self.text = re.sub(r'(?:[^\.])*', replace_w_blank, self.text)
+
+    def remove_newline_in_middle_of_word(self):
+        self.text = Text(self.text).apply(cr.NewLineInMiddleOfWordRule)
+
+    def replace_double_newlines(self):
+        self.text = Text(self.text).apply(cr.DoubleNewLineWithSpaceRule,
+                                          cr.DoubleNewLineRule)
+
+    def remove_pdf_line_breaks(self):
+        self.text = Text(
+                self.text).apply(cr.NewLineFollowedByBulletRule,
+                                 PDF.NewLineInMiddleOfSentenceRule,
+                                 PDF.NewLineInMiddleOfSentenceNoSpacesRule)
+
+    def replace_newlines(self):
+        if self.doc_type == 'pdf':
+            self.remove_pdf_line_breaks()
+        else:
+            self.text = Text(
+                self.text).apply(cr.NewLineFollowedByPeriodRule,
+                                 cr.ReplaceNewlineWithCarriageReturnRule)
+
+    def replace_escaped_newlines(self):
+        self.text = Text(
+                self.text).apply(cr.EscapedNewLineRule,
+                                 cr.EscapedCarriageReturnRule,
+                                 cr.TypoEscapedNewLineRule,
+                                 cr.TypoEscapedCarriageReturnRule)
+
+    def replace_punctuation_in_brackets(self):
+        def replace_punct(match):
+            match = match.group()
+            if '?' in match:
+                sub = re.sub(re.escape('?'), '&ᓷ&', match)
+                return sub
+            return match
+        self.text = re.sub(r'\[(?:[^\]])*\]', replace_punct, self.text)
+
+    def clean_quotations(self):
+        # method added explicitly
+        # pragmatic-segmenter applies thhis method
+        # at different location
+        self.text = re.sub('`', "'", self.text)
+        self.text = Text(self.text).apply(
+                                        cr.QuotationsFirstRule,
+                                        cr.QuotationsSecondRule)
+
+    def clean_table_of_contents(self):
+        self.text = Text(self.text).apply(
+                                        cr.TableOfContentsRule,
+                                        cr.ConsecutivePeriodsRule,
+                                        cr.ConsecutiveForwardSlashRule)
+
+    def search_for_connected_sentences(self, word, txt, regex, rule):
+        if not re.search(regex, word):
+            return txt
+        if any(k in word for k in cr.URL_EMAIL_KEYWORDS):
+            return txt
+        new_word = Text(word).apply(rule)
+        txt = re.sub(re.escape(word), new_word, txt)
+        return txt
+
+    def check_for_no_space_in_between_sentences(self):
+        words = self.text.split(' ')
+        for word in words:
+            self.text = self.search_for_connected_sentences(word, self.text, cr.NO_SPACE_BETWEEN_SENTENCES_REGEX, cr.NoSpaceBetweenSentencesRule)
+            self.text = self.search_for_connected_sentences(word, self.text, cr.NO_SPACE_BETWEEN_SENTENCES_DIGIT_REGEX, cr.NoSpaceBetweenSentencesDigitRule)
+
+    def clean_consecutive_characters(self):
+        self.text = Text(self.text).apply(
+                                        cr.ConsecutivePeriodsRule,
+                                        cr.ConsecutiveForwardSlashRule)

pysbd/exclamation_words.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+import re
+from pysbd.punctuation_replacer import replace_punctuation
+
+
+class ExclamationWords(object):
+    """
+    Searches for exclamation points that are part of words
+    and not ending punctuation and replaces them.
+    """
+    EXCLAMATION_WORDS = "!Xũ !Kung ǃʼOǃKung !Xuun !Kung-Ekoka ǃHu ǃKhung ǃKu ǃung ǃXo ǃXû ǃXung ǃXũ !Xun Yahoo! Y!J Yum!".split()
+    EXCLAMATION_REGEX = r"|".join(re.escape(w) for w in EXCLAMATION_WORDS)
+
+    @classmethod
+    def apply_rules(cls, text):
+        return re.sub(ExclamationWords.EXCLAMATION_REGEX, replace_punctuation,
+                      text)

pysbd/lang/amharic.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+from pysbd.abbreviation_replacer import AbbreviationReplacer
+from pysbd.lang.common import Common, Standard
+
+class Amharic(Common, Standard):
+
+    iso_code = 'am'
+
+    SENTENCE_BOUNDARY_REGEX = r'.*?[፧።!\?]|.*?$'
+    Punctuations = ['።', '፧', '?', '!']
+
+    class AbbreviationReplacer(AbbreviationReplacer):
+        SENTENCE_STARTERS = []

pysbd/lang/arabic.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+import re
+
+from pysbd.abbreviation_replacer import AbbreviationReplacer
+from pysbd.lang.common import Common, Standard
+from pysbd.utils import Rule
+
+class Arabic(Common, Standard):
+
+    iso_code = 'ar'
+
+    Punctuations = ['?', '!', ':', '.', '؟', '،']
+    SENTENCE_BOUNDARY_REGEX = r'.*?[:\.!\?؟،]|.*?\Z|.*?$'
+
+    # Rubular: http://rubular.com/r/RX5HpdDIyv
+    ReplaceColonBetweenNumbersRule = Rule(r'(?<=\d):(?=\d)', '♭')
+
+    # Rubular: http://rubular.com/r/kPRgApNHUg
+    ReplaceNonSentenceBoundaryCommaRule = Rule(r'،(?=\s\S+،)', '♬')
+
+    class AbbreviationReplacer(AbbreviationReplacer):
+
+        SENTENCE_STARTERS = []
+
+        def __init__(self, text, lang):
+            super().__init__(text, lang)
+
+        def scan_for_replacements(self, txt, am, index, character_array):
+            txt = re.sub('(?<={0})\.'.format(am), '∯', txt)
+            return txt
+
+    class Abbreviation(Standard.Abbreviation):
+        ABBREVIATIONS = ['ا', 'ا. د', 'ا.د', 'ا.ش.ا', 'ا.ش.ا', 'إلخ', 'ت.ب', 'ت.ب', 'ج.ب', 'جم', 'ج.ب', 'ج.م.ع', 'ج.م.ع', 'س.ت', 'س.ت', 'سم', 'ص.ب.', 'ص.ب', 'كج.', 'كلم.', 'م', 'م.ب', 'م.ب', 'ه',]
+        PREPOSITIVE_ABBREVIATIONS = []
+        NUMBER_ABBREVIATIONS = []

pysbd/lang/armenian.py

@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*-
+from pysbd.abbreviation_replacer import AbbreviationReplacer
+from pysbd.lang.common import Common, Standard
+from pysbd.between_punctuation import BetweenPunctuation
+import re
+from functools import partial
+from pysbd.punctuation_replacer import replace_punctuation
+
+class Armenian(Common, Standard):
+
+    iso_code = 'hy'
+
+    SENTENCE_BOUNDARY_REGEX = r"（(?:[^）])*）(?=\s?[Ա-ՖA-Z])|「(?:[^」])*」(?=\s[Ա-ՖA-Z])|\((?:[^\)]){2,}\)(?=\s[Ա-ՖA-Z])|\'(?:[^\'])*[^,]\'(?=\s[Ա-ՖA-Z])|\"(?:[^\"])*[^,]\"(?=\s[Ա-ՖA-Z])|\“(?:[^\”])*[^,]\”(?=\s[Ա-ՖA-Z])|[。．.！!?？ ]{2,}|\S.*?[。．.！!?？ȸȹ☉☈☇☄]|[。．.！!?？]|.*?(?<!\d)[։]"
+
+
+    # SENTENCE_BOUNDARY_REGEX = r'（(?:[^）])*）(?=\s?[Ա-ՖA-Z0-9])|.*?(?<!\d)[։]|.*?$'
+    Punctuations = ['։']
+
+
+    QUOTATION_AT_END_OF_SENTENCE_REGEX = r'[.․։][\"\'“”»«]\s{1}[A-ZԱ-Ֆ]'
+
+    SPLIT_SPACE_QUOTATION_AT_END_OF_SENTENCE_REGEX = r'(?<=[.․։][\"\'“”»«])\s{1}(?=[A-ZԱ-Ֆ])'
+
+    class AbbreviationReplacer(AbbreviationReplacer):
+        SENTENCE_STARTERS = []
+
+    class BetweenPunctuation(BetweenPunctuation):
+        BETWEEN_SINGLE_QUOTES_ARMENIAN_REGEX = r"(?<=\s)'(?:[^']|'[ա-ֆԱ-Ֆ])*'"
+
+        BETWEEN_SINGLE_QUOTE_SLANTED_ARMENIAN_REGEX = r"(?<=\s)‘(?:[^’]|’[ա-ֆԱ-Ֆ])*’"
+
+        BETWEEN_DOUBLE_QUOTES_REGEX = r'"(?>[^"\\]+|\\{2}|\\.)*"'
+
+        BETWEEN_DOUBLE_QUOTES_ARMENIAN_REGEX_2 = r'"(?=(?P<tmp>[^\"\\]+|\\{2}|\\.)*)(?P=tmp)"'
+
+        # Rubular: http://rubular.com/r/x6s4PZK8jc
+        BETWEEN_QUOTE_ARROW_ARMENIAN_REGEX = r'«(?>[^»\\]+|\\{2}|\\.)*»'
+
+        BETWEEN_QUOTE_ARROW_ARMENIAN_REGEX_2 = r"\«(?=(?P<tmp>[^»\\]+|\\{2}|\\.)*)(?P=tmp)\»"
+
+        # Rubular: http://rubular.com/r/JbAIpKdlSq
+        BETWEEN_QUOTE_SLANTED_ARMENIAN_REGEX = r"“(?>[^”\\]+|\\{2}|\\.)*”"
+        BETWEEN_QUOTE_SLANTED_ARMENIAN_REGEX_2 = r"\“(?=(?P<tmp>[^”\\]+|\\{2}|\\.)*)(?P=tmp)\”"
+
+        # Rubular: http://rubular.com/r/WX4AvnZvlX
+        BETWEEN_SQUARE_BRACKETS_ARMENIAN_REGEX = r"\[(?>[^\]\\]+|\\{2}|\\.)*\]"
+
+        BETWEEN_SQUARE_BRACKETS_ARMENIAN_REGEX_2 = r'\[(?=(?P<tmp>[^\]\\]+|\\{2}|\\.)*)(?P=tmp)\]'
+
+        # Rubular: http://rubular.com/r/6tTityPflI
+        BETWEEN_PARENS_ARMENIAN_REGEX = r"\((?>[^\(\)\\]+|\\{2}|\\.)*\)"
+
+        BETWEEN_PARENS_ARMENIAN_REGEX_2 = r"\((?=(?P<tmp>[^\(\)\\]+|\\{2}|\\.)*)(?P=tmp)\)"
+
+        # Rubular: http://rubular.com/r/mXf8cW025o
+        WORD_WITH_LEADING_APOSTROPHE_ARMENIAN = r"(?<=\s)'(?:[^']|'[ա-ֆԱ-Ֆ])*'\S"
+
+        # Rubular: http://rubular.com/r/jTtDKfjxzr
+        BETWEEN_EM_DASHES_REGEX_ARMENIAN = r"\-\-(?>[^\-\-])*\-\-"
+
+        BETWEEN_EM_DASHES_REGEX_2_ARMENIAN = r"--(?=(?P<tmp>[^--]*))(?P=tmp)--"
+
+        def __init__(self, text):
+            super().__init__(text)
+
+        def replace(self):
+            text = self.sub_punctuation_between_quotes_and_parens(self.text)
+            return self.sub_punctuation_between_quotes_and_parens_armenian(text)
+
+        def sub_punctuation_between_quotes_and_parens_armenian(self, txt):
+            txt = self.sub_punctuation_between_single_quotes_armenian(txt)
+            txt = self.sub_punctuation_between_single_quote_slanted_armenian(txt)
+            txt = self.sub_punctuation_between_double_quotes_armenian(txt)
+            txt = self.sub_punctuation_between_square_brackets_armenian(txt)
+            txt = self.sub_punctuation_between_parens_armenian(txt)
+            txt = self.sub_punctuation_between_quotes_arrow_armenian(txt)
+            txt = self.sub_punctuation_between_em_dashes_armenian(txt)
+            txt = self.sub_punctuation_between_quotes_slanted_armenian(txt)
+            return txt
+
+        def sub_punctuation_between_single_quotes_armenian(self, txt):
+            if re.search(self.WORD_WITH_LEADING_APOSTROPHE_ARMENIAN, txt) and \
+                    (not re.search(r"'\s", txt)):
+                return txt
+            return re.sub(self.BETWEEN_SINGLE_QUOTES_ARMENIAN_REGEX,
+                        partial(replace_punctuation, match_type='single'), txt)
+
+        def sub_punctuation_between_single_quote_slanted_armenian(self, txt):
+            return re.sub(self.BETWEEN_SINGLE_QUOTE_SLANTED_ARMENIAN_REGEX,
+                        replace_punctuation, txt)
+
+
+        def sub_punctuation_between_parens_armenian(self, txt):
+            return re.sub(self.BETWEEN_PARENS_ARMENIAN_REGEX_2, replace_punctuation, txt)
+
+        def sub_punctuation_between_square_brackets_armenian(self, txt):
+            return re.sub(self.BETWEEN_SQUARE_BRACKETS_ARMENIAN_REGEX_2, replace_punctuation,
+                        txt)
+
+        def sub_punctuation_between_double_quotes_armenian(self, txt):
+            return re.sub(self.BETWEEN_DOUBLE_QUOTES_ARMENIAN_REGEX_2, replace_punctuation,
+                        txt)
+
+        def sub_punctuation_between_quotes_arrow_armenian(self, txt):
+            return re.sub(self.BETWEEN_QUOTE_ARROW_ARMENIAN_REGEX_2, replace_punctuation, txt)
+
+        def sub_punctuation_between_em_dashes_armenian(self, txt):
+            return re.sub(self.BETWEEN_EM_DASHES_REGEX_2_ARMENIAN, replace_punctuation, txt)
+
+        def sub_punctuation_between_quotes_slanted_armenian(self, txt):
+            return re.sub(self.BETWEEN_QUOTE_SLANTED_ARMENIAN_REGEX_2, replace_punctuation,
+                        txt)

pysbd/lang/bulgarian.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+import re
+
+from pysbd.abbreviation_replacer import AbbreviationReplacer
+from pysbd.lang.common import Common, Standard
+
+class Bulgarian(Common, Standard):
+
+    iso_code = 'bg'
+
+    class Abbreviation(Standard.Abbreviation):
+        ABBREVIATIONS = ["p.s", "акад", "ал", "б.р", "б.ред", "бел.а", "бел.пр", "бр", "бул", "в", "вж", "вкл", "вм", "вр", "г", "ген", "гр", "дж", "дм", "доц", "др", "ем", "заб", "зам", "инж", "к.с", "кв", "кв.м", "кг", "км", "кор", "куб", "куб.м", "л", "лв", "м", "м.г", "мин", "млн", "млрд", "мм", "н.с", "напр", "пл", "полк", "проф", "р", "рис", "с", "св", "сек", "см", "сп", "срв", "ст", "стр", "т", "т.г", "т.е", "т.н", "т.нар", "табл", "тел", "у", "ул", "фиг", "ха", "хил", "ч", "чл", "щ.д"]
+        NUMBER_ABBREVIATIONS = []
+        PREPOSITIVE_ABBREVIATIONS = []
+
+    class AbbreviationReplacer(AbbreviationReplacer):
+        SENTENCE_STARTERS = []
+
+        def __init__(self, text, lang):
+            super().__init__(text, lang)
+
+        def replace_period_of_abbr(self, txt, abbr):
+            txt = re.sub(r'(?<=\s{abbr})\.|(?<=^{abbr})\.'.format(abbr=abbr.strip()), '∯', txt)
+            return txt

pysbd/lang/burmese.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+from pysbd.abbreviation_replacer import AbbreviationReplacer
+from pysbd.lang.common import Common, Standard
+
+class Burmese(Common, Standard):
+
+    iso_code = 'my'
+
+    SENTENCE_BOUNDARY_REGEX = r'.*?[။၏!\?]|.*?$'
+    Punctuations = ['။', '၏', '?', '!']
+
+    class AbbreviationReplacer(AbbreviationReplacer):
+        SENTENCE_STARTERS = []

pysbd/lang/chinese.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+import re
+from pysbd.abbreviation_replacer import AbbreviationReplacer
+from pysbd.between_punctuation import BetweenPunctuation
+from pysbd.lang.common import Common, Standard
+from pysbd.punctuation_replacer import replace_punctuation
+
+class Chinese(Common, Standard):
+
+    iso_code = 'zh'
+
+    class AbbreviationReplacer(AbbreviationReplacer):
+        SENTENCE_STARTERS = []
+
+    class BetweenPunctuation(BetweenPunctuation):
+
+        def __init__(self, text):
+            super().__init__(text)
+
+        def replace(self):
+            self.sub_punctuation_between_quotes_and_parens()
+            return self.text
+
+        def sub_punctuation_between_double_angled_quotation_marks(self):
+            BETWEEN_DOUBLE_ANGLE_QUOTATION_MARK_REGEX = r"《(?=(?P<tmp>[^》\\]+|\\{2}|\\.)*)(?P=tmp)》"
+            self.text = re.sub(BETWEEN_DOUBLE_ANGLE_QUOTATION_MARK_REGEX, replace_punctuation,
+                      self.text)
+
+        def sub_punctuation_between_l_bracket(self):
+            BETWEEN_L_BRACKET_REGEX = r"「(?=(?P<tmp>[^」\\]+|\\{2}|\\.)*)(?P=tmp)」"
+            self.text = re.sub(BETWEEN_L_BRACKET_REGEX, replace_punctuation,
+                      self.text)
+
+        def sub_punctuation_between_quotes_and_parens(self):
+            self.sub_punctuation_between_double_angled_quotation_marks()
+            self.sub_punctuation_between_l_bracket()

pysbd/lang/common/__init__.py

@@ -0,0 +1,2 @@
+from .common import Common  # noqa: F401
+from .standard import Standard  # noqa: F401

pysbd/lang/common/common.py

@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+import re
+from pysbd.utils import Rule
+
+class Common(object):
+
+    # added special case: r"[。．.！!? ]{2,}" to handle intermittent dots, exclamation, etc.
+    # r"[。．.！!?] at end to handle single instances of these symbol inputs
+    SENTENCE_BOUNDARY_REGEX = r"（(?:[^）])*）(?=\s?[A-Z])|「(?:[^」])*」(?=\s[A-Z])|\((?:[^\)]){2,}\)(?=\s[A-Z])|\'(?:[^\'])*[^,]\'(?=\s[A-Z])|\"(?:[^\"])*[^,]\"(?=\s[A-Z])|\“(?:[^\”])*[^,]\”(?=\s[A-Z])|[。．.！!?？ ]{2,}|\S.*?[。．.！!?？ȸȹ☉☈☇☄]|[。．.！!?？]"
+
+    # # Rubular: http://rubular.com/r/NqCqv372Ix
+    QUOTATION_AT_END_OF_SENTENCE_REGEX = r'[!?\.-][\"\'“”]\s{1}[A-Z]'
+
+    # # Rubular: http://rubular.com/r/6flGnUMEVl
+    PARENS_BETWEEN_DOUBLE_QUOTES_REGEX = r'["\”]\s\(.*\)\s["\“]'
+
+    # # Rubular: http://rubular.com/r/TYzr4qOW1Q
+    # BETWEEN_DOUBLE_QUOTES_REGEX = / "(?:[^"])*[^, ]"|“(?: [ ^”])*[^, ]”/
+
+    # # Rubular: http://rubular.com/r/JMjlZHAT4g
+    SPLIT_SPACE_QUOTATION_AT_END_OF_SENTENCE_REGEX = r'(?<=[!?\.-][\"\'“”])\s{1}(?=[A-Z])'
+
+    # # Rubular: http://rubular.com/r/mQ8Es9bxtk
+    CONTINUOUS_PUNCTUATION_REGEX = r'(?<=\S)(!|\?){3,}(?=(\s|\Z|$))'
+
+    # https://rubular.com/r/UkumQaILKbkeyc
+    # https://github.com/diasks2/pragmatic_segmenter/commit/d9ec1a352aff92b91e2e572c30bb9561eb42c703
+    NUMBERED_REFERENCE_REGEX = r'(?<=[^\d\s])(\.|∯)((\[(\d{1,3},?\s?-?\s?)*\b\d{1,3}\])+|((\d{1,3}\s?)?\d{1,3}))(\s)(?=[A-Z])'
+
+    # # Rubular: http://rubular.com/r/yqa4Rit8EY
+    PossessiveAbbreviationRule = Rule(r"\.(?='s\s)|\.(?='s$)|\.(?='s\Z)", '∯')
+
+    # # Rubular: http://rubular.com/r/NEv265G2X2
+    KommanditgesellschaftRule = Rule(r'(?<=Co)\.(?=\sKG)', '∯')
+
+    # # Rubular: http://rubular.com/r/xDkpFZ0EgH
+    MULTI_PERIOD_ABBREVIATION_REGEX = r"\b[a-z](?:\.[a-z])+[.]"
+
+    class SingleLetterAbbreviationRules(object):
+        """Searches for periods within an abbreviation and
+        replaces the periods.
+        """
+        # Rubular: http://rubular.com/r/e3H6kwnr6H
+        SingleUpperCaseLetterAtStartOfLineRule = Rule(r"(?<=^[A-Z])\.(?=\s)", '∯')
+
+        # Rubular: http://rubular.com/r/gitvf0YWH4
+        SingleUpperCaseLetterRule = Rule(r"(?<=\s[A-Z])\.(?=,?\s)", '∯')
+
+        All = [
+            SingleUpperCaseLetterAtStartOfLineRule, SingleUpperCaseLetterRule
+        ]
+
+    class AmPmRules(object):
+
+        # Rubular: http://rubular.com/r/Vnx3m4Spc8
+        UpperCasePmRule = Rule(r'(?<= P∯M)∯(?=\s[A-Z])', '.')
+
+        # Rubular: http://rubular.com/r/AJMCotJVbW
+        UpperCaseAmRule = Rule(r'(?<=A∯M)∯(?=\s[A-Z])', '.')
+
+        # Rubular: http://rubular.com/r/13q7SnOhgA
+        LowerCasePmRule = Rule(r'(?<=p∯m)∯(?=\s[A-Z])', '.')
+
+        # Rubular: http://rubular.com/r/DgUDq4mLz5
+        LowerCaseAmRule = Rule(r'(?<=a∯m)∯(?=\s[A-Z])', '.')
+
+        All = [UpperCasePmRule, UpperCaseAmRule, LowerCasePmRule, LowerCaseAmRule]
+
+    class Numbers(object):
+        # Rubular: http://rubular.com/r/oNyxBOqbyy
+        PeriodBeforeNumberRule = Rule(r'\.(?=\d)', '∯')
+
+        # Rubular: http://rubular.com/r/EMk5MpiUzt
+        NumberAfterPeriodBeforeLetterRule = Rule(r'(?<=\d)\.(?=\S)', '∯')
+
+        # Rubular: http://rubular.com/r/rf4l1HjtjG
+        NewLineNumberPeriodSpaceLetterRule = Rule(r'(?<=\r\d)\.(?=(\s\S)|\))', '∯')
+
+        # Rubular: http://rubular.com/r/HPa4sdc6b9
+        StartLineNumberPeriodRule = Rule(r'(?<=^\d)\.(?=(\s\S)|\))', '∯')
+
+        # Rubular: http://rubular.com/r/NuvWnKleFl
+        StartLineTwoDigitNumberPeriodRule = Rule(r'(?<=^\d\d)\.(?=(\s\S)|\))', '∯')
+
+        All = [
+            PeriodBeforeNumberRule,
+            NumberAfterPeriodBeforeLetterRule,
+            NewLineNumberPeriodSpaceLetterRule,
+            StartLineNumberPeriodRule,
+            StartLineTwoDigitNumberPeriodRule
+            ]

pysbd/lang/common/standard.py

@@ -0,0 +1,114 @@
+# -*- coding: utf-8 -*-
+from pysbd.utils import Rule
+from pysbd.abbreviation_replacer import AbbreviationReplacer
+
+class Standard:
+
+    # This class holds the punctuation marks.
+    Punctuations = ['。', '．', '.', '！', '!', '?', '？']
+
+    # Rubular: http://rubular.com/r/G2opjedIm9
+    GeoLocationRule = Rule(r'(?<=[a-zA-z]°)\.(?=\s*\d+)', '∯')
+
+    FileFormatRule = Rule(r'(?<=\s)\.(?=(jpe?g|png|gif|tiff?|pdf|ps|docx?|xlsx?|svg|bmp|tga|exif|odt|html?|txt|rtf|bat|sxw|xml|zip|exe|msi|blend|wmv|mp[34]|pptx?|flac|rb|cpp|cs|js)\s)', '∯')
+
+    SingleNewLineRule = Rule(r'\n', 'ȹ')
+
+    # Rubular: http://rubular.com/r/aXPUGm6fQh
+    QuestionMarkInQuotationRule = Rule(r'\?(?=(\'|\"))', '&ᓷ&')
+
+    ExtraWhiteSpaceRule = Rule(r'\s{3,}', ' ')
+
+    SubSingleQuoteRule = Rule(r'&⎋&', "'")
+
+    class Abbreviation(object):
+        """Defines the abbreviations for each language (if available)"""
+        ABBREVIATIONS = ['adj', 'adm', 'adv', 'al', 'ala', 'alta', 'apr', 'arc', 'ariz', 'ark', 'art', 'assn', 'asst', 'attys', 'aug', 'ave', 'bart', 'bld', 'bldg', 'blvd', 'brig', 'bros', 'btw', 'cal', 'calif', 'capt', 'cl', 'cmdr', 'co', 'col', 'colo', 'comdr', 'con', 'conn', 'corp', 'cpl', 'cres', 'ct', 'd.phil', 'dak', 'dec', 'del', 'dept', 'det', 'dist', 'dr', 'dr.phil', 'dr.philos', 'drs', 'e.g', 'ens', 'esp', 'esq', 'etc', 'exp', 'expy', 'ext', 'feb', 'fed', 'fla', 'ft', 'fwy', 'fy', 'ga', 'gen', 'gov', 'hon', 'hosp', 'hr', 'hway', 'hwy', 'i.e', 'ia', 'id', 'ida', 'ill', 'inc', 'ind', 'ing', 'insp', 'is', 'jan', 'jr', 'jul', 'jun', 'kan', 'kans', 'ken', 'ky', 'la', 'lt', 'ltd', 'maj', 'man', 'mar', 'mass', 'may', 'md', 'me', 'med', 'messrs', 'mex', 'mfg', 'mich', 'min', 'minn', 'miss', 'mlle', 'mm', 'mme', 'mo', 'mont', 'mr', 'mrs', 'ms', 'msgr', 'mssrs', 'mt', 'mtn', 'neb', 'nebr', 'nev', 'no', 'nos', 'nov', 'nr', 'oct', 'ok', 'okla', 'ont', 'op', 'ord', 'ore', 'p', 'pa', 'pd', 'pde', 'penn', 'penna', 'pfc', 'ph', 'ph.d', 'pl', 'plz', 'pp', 'prof', 'pvt', 'que', 'rd', 'rs', 'ref', 'rep', 'reps', 'res', 'rev', 'rt', 'sask', 'sec', 'sen', 'sens', 'sep', 'sept', 'sfc', 'sgt', 'sr', 'st', 'supt', 'surg', 'tce', 'tenn', 'tex', 'univ', 'usafa', 'u.s', 'ut', 'va', 'v', 'ver', 'viz', 'vs', 'vt', 'wash', 'wis', 'wisc', 'wy', 'wyo', 'yuk', 'fig']
+        PREPOSITIVE_ABBREVIATIONS = ['adm', 'attys', 'brig', 'capt', 'cmdr', 'col', 'cpl', 'det', 'dr', 'gen', 'gov', 'ing', 'lt', 'maj', 'mr', 'mrs', 'ms', 'mt', 'messrs', 'mssrs', 'prof', 'ph', 'rep', 'reps', 'rev', 'sen', 'sens', 'sgt', 'st', 'supt', 'v', 'vs', 'fig']
+        NUMBER_ABBREVIATIONS = ['art', 'ext', 'no', 'nos', 'p', 'pp']
+
+        # Rubular: http://rubular.com/r/EUbZCNfgei
+        # WithMultiplePeriodsAndEmailRule = Rule(r'(\w)(\.)(\w)', '\\1∮\\3')
+        # \w in python matches unicode abbreviations also so limit to english alphanumerics
+        WithMultiplePeriodsAndEmailRule = Rule(r'([a-zA-Z0-9_])(\.)([a-zA-Z0-9_])', '\\1∮\\3')
+
+    class DoublePunctuationRules(object):
+        FirstRule = Rule(r'\?!', '☉')
+        SecondRule = Rule(r'!\?', '☈')
+        ThirdRule = Rule(r'\?\?', '☇')
+        ForthRule = Rule(r'!!', '☄')
+        DoublePunctuation = r'\?!|!\?|\?\?|!!'
+        All = [FirstRule, SecondRule, ThirdRule, ForthRule]
+
+    class ExclamationPointRules(object):
+        # Rubular: http://rubular.com/r/XS1XXFRfM2
+        InQuotationRule = Rule(r'\!(?=(\'|\"))', '&ᓴ&')
+
+        # Rubular: http://rubular.com/r/sl57YI8LkA
+        BeforeCommaMidSentenceRule = Rule(r'\!(?=\,\s[a-z])', '&ᓴ&')
+
+        # Rubular: http://rubular.com/r/f9zTjmkIPb
+        MidSentenceRule = Rule(r'\!(?=\s[a-z])', '&ᓴ&')
+
+        All = [InQuotationRule, BeforeCommaMidSentenceRule, MidSentenceRule]
+
+    class SubSymbolsRules(object):
+        Period = Rule(r'∯', '.')
+        ArmenianFullStop = Rule(r'⍟', '։')
+        ArabicComma = Rule(r'♬', '،')
+        SemiColon = Rule(r'♭', ':')
+        FullWidthPeriod = Rule(r'&ᓰ&', '。')
+        SpecialPeriod = Rule(r'&ᓱ&', '．')
+        FullWidthExclamation = Rule(r'&ᓳ&', '！')
+        ExclamationPoint = Rule(r'&ᓴ&', '!')
+        QuestionMark = Rule(r'&ᓷ&', '?')
+        FullWidthQuestionMark = Rule(r'&ᓸ&', '？')
+        MixedDoubleQE = Rule(r'☉', '?!')
+        MixedDoubleQQ = Rule(r'☇', '??')
+        MixedDoubleEQ = Rule(r'☈', '!?')
+        MixedDoubleEE = Rule(r'☄', '!!')
+        LeftParens = Rule(r'&✂&', '(')
+        RightParens = Rule(r'&⌬&', ')')
+        TemporaryEndingPunctutation = Rule(r'ȸ', '')
+        Newline = Rule(r'ȹ', "\n")
+        All = [Period, ArmenianFullStop, ArabicComma, SemiColon, FullWidthPeriod, SpecialPeriod,
+               FullWidthExclamation, ExclamationPoint, QuestionMark,
+               FullWidthQuestionMark, MixedDoubleQE, MixedDoubleQQ, MixedDoubleEQ,
+               MixedDoubleEE, LeftParens, RightParens, TemporaryEndingPunctutation,
+               Newline]
+
+    class EllipsisRules(object):
+
+        # below rules aren't similar to original rules of pragmatic segmenter
+        # modification: spaces replaced with same number of symbols
+        # Rubular: http://rubular.com/r/i60hCK81fz
+        ThreeConsecutiveRule = Rule(r'\.\.\.(?=\s+[A-Z])', '☏☏.')
+
+        # Rubular: http://rubular.com/r/Hdqpd90owl
+        FourConsecutiveRule = Rule(r'(?<=\S)\.{3}(?=\.\s[A-Z])', 'ƪƪƪ')
+
+        # Rubular: http://rubular.com/r/YBG1dIHTRu
+        ThreeSpaceRule = Rule(r'(\s\.){3}\s', '♟♟♟♟♟♟♟')
+
+        # Rubular: http://rubular.com/r/2VvZ8wRbd8
+        FourSpaceRule = Rule(r'(?<=[a-z])(\.\s){3}\.($|\\n)', '♝♝♝♝♝♝♝')
+
+        OtherThreePeriodRule = Rule(r'\.\.\.', 'ƪƪƪ')
+
+        All = [ThreeSpaceRule, FourSpaceRule, FourConsecutiveRule,
+               ThreeConsecutiveRule, OtherThreePeriodRule]
+
+    class ReinsertEllipsisRules(object):
+        # below rules aren't similar to original rules of pragmatic segmenter
+        # modification: symbols replaced with same number of ellipses
+        SubThreeConsecutivePeriod = Rule(r'ƪƪƪ', '...')
+        SubThreeSpacePeriod = Rule(r'♟♟♟♟♟♟♟', ' . . . ')
+        SubFourSpacePeriod = Rule(r'♝♝♝♝♝♝♝', '. . . .')
+        SubTwoConsecutivePeriod = Rule(r'☏☏', '..')
+        SubOnePeriod = Rule(r'∮', '.')
+        All = [SubThreeConsecutivePeriod, SubThreeSpacePeriod, SubFourSpacePeriod,
+               SubTwoConsecutivePeriod, SubOnePeriod]
+
+    class AbbreviationReplacer(AbbreviationReplacer):
+        SENTENCE_STARTERS = "A Being Did For He How However I In It Millions "\
+            "More She That The There They We What When Where Who Why".split(" ")

pysbd/lang/danish.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+import re
+from re import escape
+
+from pysbd.abbreviation_replacer import AbbreviationReplacer
+from pysbd.lang.common import Common, Standard
+from pysbd.utils import Rule
+
+class Danish(Common, Standard):
+
+    iso_code = 'da'
+
+    MONTHS = ['Januar', 'Februar', 'Marts', 'April', 'Maj', 'Juni', 'Juli', 'August', 'September', 'Oktober', 'November', 'December']
+
+    class Numbers(Common.Numbers):
+
+        NumberPeriodSpaceRule = Rule(r'(?<=\s[1-9][0-9])\.(?=\s)|(?<=\s[0-9])\.(?=\s)', '∯')
+
+        NegativeNumberPeriodSpaceRule = Rule(r'(?<=\s-[1-9][0-9])\.(?=\s)|(?<=\s-[0-9])\.(?=\s)', '∯')
+
+        All = Common.Numbers.All + [NumberPeriodSpaceRule, NegativeNumberPeriodSpaceRule]
+
+    class AbbreviationReplacer(AbbreviationReplacer):
+
+        SENTENCE_STARTERS = ("At De Dem Den Der Det Du En Et For Få Gjorde Han Hun Hvad Hvem"
+                             " Hvilke Hvor Hvordan Hvorfor Hvorledes Hvornår I Jeg Mange Vi Være").split(' ')
+
+        def __init__(self, text, lang):
+            super().__init__(text, lang)
+
+        def replace_abbreviation_as_sentence_boundary(self):
+            sent_starters = "|".join((r"(?=\s{}\s)".format(word) for word in self.SENTENCE_STARTERS))
+            regex = r"(U∯S|U\.S|U∯K|E∯U|E\.U|U∯S∯A|U\.S\.A|I|i.v|s.u|s.U)∯({})".format(sent_starters)
+            self.text = re.sub(regex, '\\1.', self.text)
+            return self.text
+
+    class Abbreviation(Standard.Abbreviation):
+        ABBREVIATIONS = ['adm', 'adr', 'afd', 'afs', 'al', 'alm', 'alm', 'ang', 'ank', 'anm', 'ann', 'ansvh', 'apr', 'arr', 'ass', 'att', 'aud', 'aug', 'aut', 'bd', 'bdt', 'bet', 'bhk', 'bio', 'biol', 'bk', 'bl.a', 'bot', 'br', 'bto', 'ca', 'cal', 'cirk', 'cit', 'co', 'cpr-nr', 'cvr-nr', 'd.d', 'd.e', 'd.m', 'd.s', 'd.s.s', 'd.y', 'd.å', 'd.æ', 'da', 'dav', 'dec', 'def', 'del', 'dep', 'diam', 'din', 'dir', 'disp', 'distr', 'do', 'dobb', 'dr', 'ds', 'dvs', 'e.b', 'e.kr', 'e.l', 'e.o', 'e.v.t', 'eftf', 'eftm', 'egl', 'eks', 'eksam', 'ekskl', 'eksp', 'ekspl', 'el', 'emer', 'endv', 'eng', 'enk', 'etc', 'eur', 'evt', 'exam', 'f', 'f', 'f.eks', 'f.kr', 'f.m', 'f.n', 'f.o', 'f.o.m', 'f.s.v', 'f.t', 'f.v.t', 'f.å', 'fa', 'fakt', 'feb', 'fec', 'ff', 'fg', 'fg', 'fhv', 'fig', 'fl', 'flg', 'fm', 'fm', 'fmd', 'forb', 'foreg', 'foren', 'forf', 'forh', 'fork', 'form', 'forr', 'fors', 'forsk', 'forts', 'fp', 'fr', 'frk', 'fuldm', 'fuldm', 'fung', 'fung', 'fys', 'fær', 'g', 'g.d', 'g.m', 'gd', 'gdr', 'gg', 'gh', 'gl', 'gn', 'gns', 'gr', 'grdl', 'gross', 'h.a', 'h.c', 'hdl', 'henh', 'henv', 'hf', 'hft', 'hhv', 'hort', 'hosp', 'hpl', 'hr', 'hrs', 'hum', 'i', 'i.e', 'ib', 'ibid', 'if', 'ifm', 'ill', 'indb', 'indreg', 'ing', 'inkl', 'insp', 'instr', 'isl', 'istf', 'jan', 'jf', 'jfr', 'jnr', 'jr', 'jul', 'jun', 'jur', 'jvf', 'kal', 'kap', 'kat', 'kbh', 'kem', 'kgl', 'kin', 'kl', 'kld', 'km/t', 'knsp', 'komm', 'kons', 'korr', 'kp', 'kr', 'kr', 'kst', 'kt', 'ktr', 'kv', 'kvt', 'l', 'l.c', 'lab', 'lat', 'lb', 'lb.', 'lb.nr', 'lejl', 'lgd', 'lic', 'lign', 'lin', 'ling.merc', 'litt', 'lok', 'lrs', 'ltr', 'lø', 'm', 'm.a.o', 'm.fl.st', 'm.m', 'm/', 'ma', 'mag', 'maks', 'mar', 'mat', 'matr.nr', 'md', 'mdl', 'mdr', 'mdtl', 'med', 'medd', 'medflg', 'medl', 'merc', 'mezz', 'mf', 'mfl', 'mgl', 'mhp', 'mht', 'mi', 'mia', 'mio', 'ml', 'mods', 'modsv', 'modt', 'mr', 'mrk', 'mrs', 'ms', 'mul', 'mv', 'mvh', 'n', 'n.br', 'n.f', 'nat', 'ned', 'nedenn', 'nedenst', 'nederl', 'nkr', 'nl', 'no', 'nord', 'nov', 'nr', 'nr', 'nto', 'nuv', 'o', 'o.a', 'o.fl.st', 'o.g', 'o.h', 'o.m.a', 'obj', 'obl', 'obs', 'odont', 'oecon', 'off', 'ofl', 'okt', 'omg', 'omr', 'omtr', 'on', 'op.cit', 'opg', 'opl', 'opr', 'org', 'orig', 'osfr', 'osv', 'ovenn', 'ovenst', 'overs', 'ovf', 'oz', 'p', 'p.a', 'p.b.v', 'p.c', 'p.m.v', 'p.p', 'p.s', 'p.t', 'p.v.a', 'p.v.c', 'par', 'partc', 'pass', 'pct', 'pd', 'pens', 'perf', 'pers', 'pg', 'pga', 'pgl', 'ph', 'ph.d', 'pharm', 'phil', 'pinx', 'pk', 'pkt', 'pl', 'pluskv', 'polit', 'polyt', 'port', 'pos', 'pp', 'pr', 'prc', 'priv', 'prod', 'prof', 'pron', 'præd', 'præf', 'præp', 'præs', 'præt', 'psych', 'pt', 'pæd', 'q.e.d', 'rad', 'red', 'ref', 'reg', 'regn', 'rel', 'rep', 'repr', 'rest', 'rk', 'russ', 's', 's.br', 's.d', 's.e', 's.f', 's.m.b.a', 's.u', 's.å', 's/', 'sa', 'sb', 'sc', 'scient', 'sek', 'sek', 'sekr', 'sem', 'sen', 'sep', 'sept', 'sg', 'sign', 'sj', 'skr', 'skt', 'slutn', 'sml', 'smp', 'sms', 'smst', 'soc', 'soc', 'sort', 'sp', 'spec', 'spm', 'spr', 'spsk', 'st', 'stk', 'str', 'stud', 'subj', 'subst', 'suff', 'sup', 'suppl', 'sv', 'såk', 'sædv', 'sø', 't', 't.h', 't.o.m', 't.v', 'tab', 'td', 'tdl', 'tdr', 'techn', 'tekn', 'temp', 'th', 'ti', 'tidl', 'tilf', 'tilh', 'till', 'tilsv', 'tjg', 'tlf', 'tlgr', 'to', 'tr', 'trp', 'tv', 'ty', 'u', 'u.p', 'u.st', 'u.å', 'uafh', 'ubf', 'ubøj', 'udb', 'udbet', 'udd', 'udg', 'uds', 'ugtl', 'ulin', 'ult', 'undt', 'univ', 'v.f', 'var', 'vb', 'vbsb', 'vedk', 'vedl', 'vedr', 'vejl', 'vh', 'vol', 'vs', 'vsa', 'vær', 'zool', 'årg', 'årh', 'årl', 'ø.f', 'øv', 'øvr']
+        NUMBER_ABBREVIATIONS = ['nr', 's']
+        PREPOSITIVE_ABBREVIATIONS = ['adm', 'skt', 'dr', 'hr', 'fru', 'st']

pysbd/lang/deutsch.py

@@ -0,0 +1,97 @@
+# -*- coding: utf-8 -*-
+import re
+from pysbd.abbreviation_replacer import AbbreviationReplacer
+from pysbd.between_punctuation import BetweenPunctuation
+from pysbd.lang.common import Common, Standard
+from pysbd.punctuation_replacer import replace_punctuation
+from pysbd.processor import Processor
+from pysbd.utils import Text, Rule
+
+
+class Deutsch(Common, Standard):
+
+    iso_code = 'de'
+
+    class Numbers(Common.Numbers):
+        # Rubular: http://rubular.com/r/hZxoyQwKT1
+        NumberPeriodSpaceRule = Rule(r'(?<=\s\d)\.(?=\s)|(?<=\s\d\d)\.(?=\s)', '∯')
+
+        # Rubular: http://rubular.com/r/ityNMwdghj
+        NegativeNumberPeriodSpaceRule = Rule(r'(?<=-\d)\.(?=\s)|(?<=-\d\d)\.(?=\s)', '∯')
+
+        All = Common.Numbers.All + [NumberPeriodSpaceRule, NegativeNumberPeriodSpaceRule]
+
+    class Processor(Processor):
+
+        def __init__(self, text, lang, char_span=False):
+            super().__init__(text, lang, char_span)
+
+        def replace_numbers(self):
+            self.text = Text(self.text).apply(*self.lang.Numbers.All)
+            self.replace_period_in_deutsch_dates()
+            return self.text
+
+        def replace_period_in_deutsch_dates(self):
+            MONTHS = ['Januar', 'Februar', 'März', 'April', 'Mai', 'Juni', 'Juli', 'August',
+                      'September', 'Oktober', 'November', 'Dezember']
+            for month in MONTHS:
+                # Rubular: http://rubular.com/r/zlqgj7G5dA
+                self.text = re.sub(r'(?<=\d)\.(?=\s*{month})'.format(month=month), '∯', self.text)
+
+    class Abbreviation(Standard.Abbreviation):
+        ABBREVIATIONS = ['Ä', 'ä', 'adj', 'adm', 'adv', 'art', 'asst', 'b.a', 'b.s', 'bart', 'bldg', 'brig', 'bros', 'bse', 'buchst', 'bzgl', 'bzw', 'c.-à-d', 'ca', 'capt', 'chr', 'cmdr', 'co', 'col', 'comdr', 'con', 'corp', 'cpl', 'd.h', 'd.j', 'dergl', 'dgl', 'dkr', 'dr ', 'ens', 'etc', 'ev ', 'evtl', 'ff', 'g.g.a', 'g.u', 'gen', 'ggf', 'gov', 'hon', 'hosp', 'i.f', 'i.h.v', 'ii', 'iii', 'insp', 'iv', 'ix', 'jun', 'k.o', 'kath ', 'lfd', 'lt', 'ltd', 'm.e', 'maj', 'med', 'messrs', 'mio', 'mlle', 'mm', 'mme', 'mr', 'mrd', 'mrs', 'ms', 'msgr', 'mwst', 'no', 'nos', 'nr', 'o.ä', 'op', 'ord', 'pfc', 'ph', 'pp', 'prof', 'pvt', 'rep', 'reps', 'res', 'rev', 'rt', 's.p.a', 'sa', 'sen', 'sens', 'sfc', 'sgt', 'sog', 'sogen', 'spp', 'sr', 'st', 'std', 'str  ', 'supt', 'surg', 'u.a  ', 'u.e', 'u.s.w', 'u.u', 'u.ä', 'usf', 'usw', 'v', 'vgl', 'vi', 'vii', 'viii', 'vs', 'x', 'xi', 'xii', 'xiii', 'xiv', 'xix', 'xv', 'xvi', 'xvii', 'xviii', 'xx', 'z.b', 'z.t', 'z.z', 'z.zt', 'zt', 'zzt', 'univ.-prof', 'o.univ.-prof', 'ao.univ.prof', 'ass.prof', 'hon.prof', 'univ.-doz', 'univ.ass', 'stud.ass', 'projektass', 'ass', 'di', 'dipl.-ing', 'mag']
+        PREPOSITIVE_ABBREVIATIONS = []
+        NUMBER_ABBREVIATIONS = ['art', 'ca', 'no', 'nos', 'nr', 'pp']
+
+    class AbbreviationReplacer(AbbreviationReplacer):
+
+        SENTENCE_STARTERS = ("Am Auch Auf Bei Da Das Der Die Ein Eine Es Für Heute Ich Im In "
+          "Ist Jetzt Mein Mit Nach So Und Warum Was Wenn Wer Wie Wir").split(' ')
+
+        def __init__(self, text, lang):
+            super().__init__(text, lang)
+
+        def replace(self):
+            # Rubular: http://rubular.com/r/B4X33QKIL8
+            SingleLowerCaseLetterRule = Rule(r'(?<=\s[a-z])\.(?=\s)', '∯')
+
+            # Rubular: http://rubular.com/r/iUNSkCuso0
+            SingleLowerCaseLetterAtStartOfLineRule = Rule(r'(?<=^[a-z])\.(?=\s)', '∯')
+            self.text = Text(self.text).apply(
+                    self.lang.PossessiveAbbreviationRule,
+                    *self.lang.SingleLetterAbbreviationRules.All,
+                    SingleLowerCaseLetterRule,
+                    SingleLowerCaseLetterAtStartOfLineRule)
+
+            self.text = self.search_for_abbreviations_in_string(self.text)
+            self.replace_multi_period_abbreviations()
+            self.text = Text(self.text).apply(*self.lang.AmPmRules.All)
+            self.text = self.replace_abbreviation_as_sentence_boundary()
+            return self.text
+
+        def scan_for_replacements(self, txt, am, index, character_array):
+            txt = re.sub(r'(?<={am})\.(?=\s)'.format(am=am), '∯', txt)
+            return txt
+
+    class BetweenPunctuation(BetweenPunctuation):
+
+        def __init__(self, text):
+            super().__init__(text)
+
+        def sub_punctuation_between_double_quotes(self, txt):
+            # Rubular: http://rubular.com/r/OdcXBsub0w
+            BETWEEN_UNCONVENTIONAL_DOUBLE_QUOTE_DE_REGEX = r',,(?=(?P<tmp>[^“\\]+|\\{2}|\\.)*)(?P=tmp)“'
+
+            # Rubular: http://rubular.com/r/2UskIupGgP
+            # SPLIT_DOUBLE_QUOTES_DE_REGEX = r'\A„(?=(?P<tmp>[^“\\]+|\\{2}|\\.)*)(?P=tmp)“'
+
+            # Rubular: http://rubular.com/r/TkZomF9tTM
+            BETWEEN_DOUBLE_QUOTES_DE_REGEX = r'„(?=(?P<tmp>[^“\\]+|\\{2}|\\.)*)(?P=tmp)“'
+
+            if '„' in txt:
+                return re.sub(BETWEEN_DOUBLE_QUOTES_DE_REGEX, replace_punctuation, txt)
+            elif ',,' in txt:
+                return re.sub(BETWEEN_UNCONVENTIONAL_DOUBLE_QUOTE_DE_REGEX,
+                              replace_punctuation, txt)
+            else:
+                return txt

pysbd/lang/dutch.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+from pysbd.abbreviation_replacer import AbbreviationReplacer
+from pysbd.lang.common import Common, Standard
+
+class Dutch(Common, Standard):
+
+    iso_code = 'nl'
+
+    class Abbreviation(Standard.Abbreviation):
+        ABBREVIATIONS = ['a.2d', 'a.a', 'a.a.j.b', 'a.f.t', 'a.g.j.b', 'a.h.v', 'a.h.w', 'a.hosp', 'a.i', 'a.j.b', 'a.j.t', 'a.m', 'a.m.r', 'a.p.m', 'a.p.r', 'a.p.t', 'a.s', 'a.t.d.f', 'a.u.b', 'a.v.a', 'a.w', 'aanbev', 'aanbev.comm', 'aant', 'aanv.st', 'aanw', 'vnw', 'aanw.vnw', 'abd', 'abm', 'abs', 'acc.& fisc', 'acc.act', 'acc.bedr.m', 'acc.bedr.t', "acc.thema's m.", 'acc.thema’s m', 'achterv', 'act.dr', 'act.dr.fam', 'act.fisc', 'act.soc', 'adm.akk', 'adm.besl', 'adm.lex', 'adm.onderr', 'adm.ov', 'adv', 'adv', 'gen', 'adv.bl', 'afd', 'afl', 'aggl.verord', 'agr', 'al', 'alg', 'alg.richts', 'amén', 'ann.dr', 'ann.dr.lg', 'ann.dr.sc.pol', 'ann.ét.eur', 'ann.fac.dr.lg', 'ann.jur.créd', 'ann.jur.créd.règl.coll', 'ann.not', 'ann.parl', 'ann.prat.comm', 'app', 'arb', 'aud', 'arbbl', 'arbh', 'arbit.besl', 'arbrb', 'arr', 'arr.cass', 'arr.r.v.st', 'arr.verbr', 'arrondrb', 'art', 'artw', 'aud', 'b', 'b', 'en w', 'b.&w', 'b.a', 'b.a.s', 'b.b.o', 'b.best.dep', 'b.br.ex', 'b.coll.fr.gem.comm', 'b.coll.vl.gem.comm', 'b.d.cult.r', 'b.d.gem.ex', 'b.d.gem.reg', 'b.dep', 'b.e.b', 'b.f.r', 'b.fr.gem.ex', 'b.fr.gem.reg', 'b.i.h', 'b.inl.j.d', 'b.inl.s.reg', 'b.j', 'b.l', 'b.lid br.ex', 'b.lid d.gem.ex', 'b.lid fr.gem.ex', 'b.lid vl.ex', 'b.lid w.gew.ex', 'b.o.z', 'b.prov.r', 'b.r.h', 'b.s', 'b.sr', 'b.stb', 'b.t.i.r', 'b.t.s.z', 'b.t.w.rev', 'b.v', 'b.ver.coll.gem.gem.comm', 'b.verg.r.b', 'b.versl', 'b.vl.ex', 'b.voorl.reg', 'b.w', 'b.w.gew.ex', 'b.z.d.g', 'b.z.v', 'bab', 'bank fin', 'bank fin.r', 'bedr.org', 'begins', 'beheersov', 'bekendm.comm', 'bel', 'bel.besch', 'bel.w.p', 'beleidsov', 'belg', 'grondw', 'benelux jur', 'ber', 'ber.w', 'besch', 'besl', 'beslagr', 'besluitwet nr', 'bestuurswet', 'bet', 'betr', 'betr', 'vnw', 'bevest', 'bew', 'bijbl', 'ind', 'eig', 'bijbl.n.bijdr', 'bijl', 'bijv', 'bijw', 'bijz.decr', 'bin.b', 'bkh', 'bl', 'blz', 'bm', 'bn', 'bnlx merkw', 'bnlx tek', 'bnlx uitl', 'rh', 'bnw', 'bouwr', 'br drs', 'br.parl', 'bs', 'bt drs', 'btw rev', 'bull', 'bull.adm.pénit', 'bull.ass', 'bull.b.m.m', 'bull.bel', 'bull.best.strafinr', 'bull.bmm', 'bull.c.b.n', 'bull.c.n.c', 'bull.cbn', 'bull.centr.arb', 'bull.cnc', 'bull.contr', 'bull.doc.min.fin', 'bull.f.e.b', 'bull.feb', 'bull.fisc.fin.r', 'bull.i.u.m', 'bull.inf.ass.secr.soc', 'bull.inf.i.e.c', 'bull.inf.i.n.a.m.i', 'bull.inf.i.r.e', 'bull.inf.iec', 'bull.inf.inami', 'bull.inf.ire', 'bull.inst.arb', 'bull.ium', 'bull.jur.imm', 'bull.lég.b', 'bull.off', 'bull.trim.b.dr.comp', 'bull.us', 'bull.v.b.o', 'bull.vbo', 'bv i.o', 'bv', 'bw int.reg', 'bw', 'bxh', 'byz', 'c', 'c.& f', 'c.& f.p', 'c.a', 'c.a.-a', 'c.a.b.g', 'c.c', 'c.c.i', 'c.c.s', 'c.conc.jur', 'c.d.e', 'c.d.p.k', 'c.e', 'c.ex', 'c.f', 'c.h.a', 'c.i.f', 'c.i.f.i.c', 'c.j', 'c.l', 'c.n', 'c.o.d', 'c.p', 'c.pr.civ', 'c.q', 'c.r', 'c.r.a', 'c.s', 'c.s.a', 'c.s.q.n', 'c.v', 'c.v.a', 'c.v.o', 'ca', 'cadeaust', 'cah.const', 'cah.dr.europ', 'cah.dr.immo', 'cah.dr.jud', 'cal', '2d', 'cal', '3e', 'cal', 'rprt', 'cap', 'carg', 'cass', 'cass', 'verw', 'cert', 'cf', 'ch', 'chron', 'chron.d.s', 'chron.dr.not', 'cie', 'cie', 'verz.schr', 'cir', 'circ', 'circ.z', 'cit', 'cit.loc', 'civ', 'cl.et.b', 'cmt', 'co', 'cognoss.v', 'coll', 'v', 'b', 'colp.w', 'com', 'com', 'cas', 'com.v.min', 'comm', 'comm', 'v', 'comm.bijz.ov', 'comm.erf', 'comm.fin', 'comm.ger', 'comm.handel', 'comm.pers', 'comm.pub', 'comm.straf', 'comm.v', 'comm.v.en v', 'comm.venn', 'comm.verz', 'comm.voor', 'comp', 'compt.w', 'computerr', 'con.m', 'concl', 'concr', 'conf', 'confl.w', 'confl.w.huwbetr', 'cons', 'conv', 'coöp', 'ver', 'corr', 'corr.bl', 'cour de cass', 'cour.fisc', 'cour.immo', 'cridon', 'crim', 'cur', 'cur', 'crt', 'curs', 'd', 'd.-g', 'd.a', 'd.a.v', 'd.b.f', 'd.c', 'd.c.c.r', 'd.d', 'd.d.p', 'd.e.t', 'd.gem.r', 'd.h', 'd.h.z', 'd.i', 'd.i.t', 'd.j', 'd.l.r', 'd.m', 'd.m.v', 'd.o.v', 'd.parl', 'd.w.z', 'dact', 'dat', 'dbesch', 'dbesl', 'de advoc', 'de belg.acc', 'de burg.st', 'de gem', 'de gerechtsd', 'de venn', 'de verz', 'decr', 'decr.d', 'decr.fr', 'decr.vl', 'decr.w', 'def', 'dep.opv', 'dep.rtl', 'derg', 'desp', 'det.mag', 'deurw.regl', 'dez', 'dgl', 'dhr', 'disp', 'diss', 'div', 'div.act', 'div.bel', 'dl', 'dln', 'dnotz', 'doc', 'hist', 'doc.jur.b', 'doc.min.fin', 'doc.parl', 'doctr', 'dpl', 'dpl.besl', 'dr', 'dr.banc.fin', 'dr.circ', 'dr.inform', 'dr.mr', 'dr.pén.entr', 'dr.q.m', 'drs', 'dtp', 'dwz', 'dyn', 'e cont', 'e', 'e.a', 'e.b', 'tek.mod', 'e.c', 'e.c.a', 'e.d', 'e.e', 'e.e.a', 'e.e.g', 'e.g', 'e.g.a', 'e.h.a', 'e.i', 'e.j', 'e.m.a', 'e.n.a.c', 'e.o', 'e.p.c', 'e.r.c', 'e.r.f', 'e.r.h', 'e.r.o', 'e.r.p', 'e.r.v', 'e.s.r.a', 'e.s.t', 'e.v', 'e.v.a', 'e.w', 'e&o.e', 'ec.pol.r', 'echos log', 'econ', 'ed', 'ed(s)', 'eeg verd.v', 'eex san s', 'eff', 'eg rtl', 'eig', 'eig.mag', 'eil', 'elektr', 'enmb', 'entr.et dr', 'enz', 'err', 'et al', 'et seq', 'etc', 'etq', 'eur', 'parl', 'eur.t.s', 'eur.verd.overdracht strafv', 'ev rechtsh', 'ev uitl', 'ev', 'evt', 'ex', 'ex.crim', 'exec', 'f', 'f.a.o', 'f.a.q', 'f.a.s', 'f.i.b', 'f.j.f', 'f.o.b', 'f.o.r', 'f.o.s', 'f.o.t', 'f.r', 'f.supp', 'f.suppl', 'fa', 'facs', 'fare act', 'fasc', 'fg', 'fid.ber', 'fig', 'fin.verh.w', 'fisc', 'fisc', 'tijdschr', 'fisc.act', 'fisc.koer', 'fl', 'form', 'foro', 'it', 'fr', 'fr.cult.r', 'fr.gem.r', 'fr.parl', 'fra', 'ft', 'g', 'g.a', 'g.a.v', 'g.a.w.v', 'g.g.d', 'g.m.t', 'g.o', 'g.omt.e', 'g.p', 'g.s', 'g.v', 'g.w.w', 'geb', 'gebr', 'gebrs', 'gec', 'gec.decr', 'ged', 'ged.st', 'gedipl', 'gedr.st', 'geh', 'gem', 'gem', 'en gew', 'gem', 'en prov', 'gem.gem.comm', 'gem.st', 'gem.stem', 'gem.w', 'gem.wet, gem.wet', 'gemeensch.optr', 'gemeensch.standp', 'gemeensch.strat', 'gemeent', 'gemeent.b', 'gemeent.regl', 'gemeent.verord', 'geol', 'geopp', 'gepubl', 'ger.deurw', 'ger.w', 'gerekw', 'gereq', 'gesch', 'get', 'getr', 'gev.m', 'gev.maatr', 'gew', 'ghert', 'gir.eff.verk', 'gk', 'gr', 'gramm', 'grat.w', 'gron,opm.en leermed', 'grootb.w', 'grs', 'grur ausl', 'grur int', 'grvm', 'grw', 'gst', 'gw', 'h.a', 'h.a.v.o', 'h.b.o', 'h.e.a.o', 'h.e.g.a', 'h.e.geb', 'h.e.gestr', 'h.l', 'h.m', 'h.o', 'h.r', 'h.t.l', 'h.t.m', 'h.w.geb', 'hand', 'handelsn.w', 'handelspr', 'handelsr.w', 'handelsreg.w', 'handv', 'harv.l.rev', 'hc', 'herald', 'hert', 'herz', 'hfdst', 'hfst', 'hgrw', 'hhr', 'hist', 'hooggel', 'hoogl', 'hosp', 'hpw', 'hr', 'hr', 'ms', 'hr.ms', 'hregw', 'hrg', 'hst', 'huis.just', 'huisv.w', 'huurbl', 'hv.vn', 'hw', 'hyp.w', 'i.b.s', 'i.c', 'i.c.m.h', 'i.e', 'i.f', 'i.f.p', 'i.g.v', 'i.h', 'i.h.a', 'i.h.b', 'i.l.pr', 'i.o', 'i.p.o', 'i.p.r', 'i.p.v', 'i.pl.v', 'i.r.d.i', 'i.s.m', 'i.t.t', 'i.v', 'i.v.m', 'i.v.s', 'i.w.tr', 'i.z', 'ib', 'ibid', 'icip-ing.cons', 'iem', 'ind prop', 'indic.soc', 'indiv', 'inf', 'inf.i.d.a.c', 'inf.idac', 'inf.r.i.z.i.v', 'inf.riziv', 'inf.soc.secr', 'ing', 'ing', 'cons', 'ing.cons', 'inst', 'int', 'int', 'rechtsh', 'strafz', "int'l & comp.l.q.", 'interm', 'intern.fisc.act', 'intern.vervoerr', 'inv', 'inv', 'f', 'inv.w', 'inv.wet', 'invord.w', 'inz', 'ir', 'irspr', 'iwtr', 'j', 'j.-cl', 'j.c.b', 'j.c.e', 'j.c.fl', 'j.c.j', 'j.c.p', 'j.d.e', 'j.d.f', 'j.d.s.c', 'j.dr.jeun', 'j.j.d', 'j.j.p', 'j.j.pol', 'j.l', 'j.l.m.b', 'j.l.o', 'j.ordre pharm', 'j.p.a', 'j.r.s', 'j.t', 'j.t.d.e', 'j.t.dr.eur', 'j.t.o', 'j.t.t', 'jaarl', 'jb.hand', 'jb.kred', 'jb.kred.c.s', 'jb.l.r.b', 'jb.lrb', 'jb.markt', 'jb.mens', 'jb.t.r.d', 'jb.trd', 'jeugdrb', 'jeugdwerkg.w', 'jg', 'jis', 'jl', 'journ.jur', 'journ.prat.dr.fisc.fin', 'journ.proc', 'jrg', 'jur', 'jur.comm.fl', 'jur.dr.soc.b.l.n', 'jur.f.p.e', 'jur.fpe', 'jur.niv', 'jur.trav.brux', 'jura falc', 'jurambt', 'jv.cass', 'jv.h.r.j', 'jv.hrj', 'jw', 'k', 'k', 'en m', 'k.b', 'k.g', 'k.k', 'k.m.b.o', 'k.o.o', 'k.v.k', 'k.v.v.v', 'kadasterw', 'kaderb', 'kador', 'kbo-nr', 'kg', 'kh', 'kiesw', 'kind.bes.v', 'kkr', 'koopv', 'kr', 'krankz.w', 'ksbel', 'kt', 'ktg', 'ktr', 'kvdm', 'kw.r', 'kymr', 'kzr', 'kzw', 'l', 'l.b', 'l.b.o', 'l.bas', 'l.c', 'l.gew', 'l.j', 'l.k', 'l.l', 'l.o', 'l.r.b', 'l.u.v.i', 'l.v.r', 'l.v.w', 'l.w', "l'exp.-compt.b.", 'l’exp.-compt.b', 'landinr.w', 'landscrt', 'larcier cass', 'lat', 'law.ed', 'lett', 'levensverz', 'lgrs', 'lidw', 'limb.rechtsl', 'lit', 'litt', 'liw', 'liwet', 'lk', 'll', 'll.(l.)l.r', 'loonw', 'losbl', 'ltd', 'luchtv', 'luchtv.w', 'm', 'm', 'not', 'm.a.v.o', 'm.a.w', 'm.b', 'm.b.o', 'm.b.r', 'm.b.t', 'm.d.g.o', 'm.e.a.o', 'm.e.r', 'm.h', 'm.h.d', 'm.i.v', 'm.j.t', 'm.k', 'm.m', 'm.m.a', 'm.m.h.h', 'm.m.v', 'm.n', 'm.not.fisc', 'm.nt', 'm.o', 'm.r', 'm.s.a', 'm.u.p', 'm.v.a', 'm.v.h.n', 'm.v.t', 'm.z', 'maatr.teboekgest.luchtv', 'maced', 'mand', 'max', 'mbl.not', 'me', 'med', 'med', 'v.b.o', 'med.b.u.f.r', 'med.bufr', 'med.vbo', 'meerv', 'meetbr.w', 'mém.adm', 'mgr', 'mgrs', 'mhd', 'mi.verantw', 'mil', 'mil.bed', 'mil.ger', 'min', 'min', 'aanbev', 'min', 'circ', 'min', 'fin', 'min.j.omz', 'min.just.circ', 'mitt', 'mnd', 'mod', 'mon', 'monde ass', 'mouv.comm', 'mr', 'ms', 'muz', 'mv', 'mva ii inv', 'mva inv', 'n cont', 'n', 'chr', 'n.a', 'n.a.g', 'n.a.v', 'n.b', 'n.c', 'n.chr', 'n.d', 'n.d.r', 'n.e.a', 'n.g', 'n.h.b.c', 'n.j', 'n.j.b', 'n.j.w', 'n.l', 'n.m', 'n.m.m', 'n.n', 'n.n.b', 'n.n.g', 'n.n.k', 'n.o.m', 'n.o.t.k', 'n.rapp', 'n.tijd.pol', 'n.v', 'n.v.d.r', 'n.v.d.v', 'n.v.o.b', 'n.v.t', 'nat.besch.w', 'nat.omb', 'nat.pers', 'ned.cult.r', 'neg.verkl', 'nhd', 'nieuw arch', 'wisk', 'njcm-bull', 'nl', 'nnd', 'no', 'not.fisc.m', 'not.w', 'not.wet', 'nr', 'nrs', 'nste', 'nt', 'numism', 'o', 'o.a', 'o.b', 'o.c', 'o.g', 'o.g.v', 'o.i', 'o.i.d', 'o.m', 'o.o', 'o.o.d', 'o.o.v', 'o.p', 'o.r', 'o.regl', 'o.s', 'o.t.s', 'o.t.t', 'o.t.t.t', 'o.t.t.z', 'o.tk.t', 'o.v.t', 'o.v.t.t', 'o.v.tk.t', 'o.v.v', 'ob', 'obsv', 'octr', 'octr.gem.regl', 'octr.regl', 'oe', 'oecd mod', 'off.pol', 'ofra', 'ohd', 'omb', 'omnia frat', 'omnil', 'omz', 'on.ww', 'onderr', 'onfrank', 'onteig.w', 'ontw', 'b.w', 'onuitg', 'onz', 'oorl.w', 'op.cit', 'opin.pa', 'opm', 'or', 'ord.br', 'ord.gem', 'ors', 'orth', 'os', 'osm', 'ov', 'ov.w.i', 'ov.w.ii', 'ov.ww', 'overg.w', 'overw', 'ovkst', 'ow kadasterw', 'oz', 'p', 'p.& b', 'p.a', 'p.a.o', 'p.b.o', 'p.e', 'p.g', 'p.j', 'p.m', 'p.m.a', 'p.o', 'p.o.j.t', 'p.p', 'p.v', 'p.v.s', 'pachtw', 'pag', 'pan', 'pand.b', 'pand.pér', 'parl.gesch', 'parl.gesch', 'inv', 'parl.st', 'part.arb', 'pas', 'pasin', 'pat', 'pb.c', 'pb.l', 'pens', 'pensioenverz', 'per.ber.i.b.r', 'per.ber.ibr', 'pers', 'st', 'pft', 'pg wijz.rv', 'pk', 'pktg', 'pli jur', 'plv', 'po', 'pol', 'pol.off', 'pol.r', 'pol.w', 'politie j', 'postbankw', 'postw', 'pp', 'pr', 'preadv', 'pres', 'prf', 'prft', 'prg', 'prijz.w', 'pro jus', 'proc', 'procesregl', 'prof', 'prot', 'prov', 'prov.b', 'prov.instr.h.m.g', 'prov.regl', 'prov.verord', 'prov.w', 'publ', 'publ.cour eur.d.h', 'publ.eur.court h.r', 'pun', 'pw', 'q.b.d', 'q.e.d', 'q.q', 'q.r', 'r', 'r.a.b.g', 'r.a.c.e', 'r.a.j.b', 'r.b.d.c', 'r.b.d.i', 'r.b.s.s', 'r.c', 'r.c.b', 'r.c.d.c', 'r.c.j.b', 'r.c.s.j', 'r.cass', 'r.d.c', 'r.d.i', 'r.d.i.d.c', 'r.d.j.b', 'r.d.j.p', 'r.d.p.c', 'r.d.s', 'r.d.t.i', 'r.e', 'r.f.s.v.p', 'r.g.a.r', 'r.g.c.f', 'r.g.d.c', 'r.g.f', 'r.g.z', 'r.h.a', 'r.i.c', 'r.i.d.a', 'r.i.e.j', 'r.i.n', 'r.i.s.a', 'r.j.d.a', 'r.j.i', 'r.k', 'r.l', 'r.l.g.b', 'r.med', 'r.med.rechtspr', 'r.n.b', 'r.o', 'r.orde apoth', 'r.ov', 'r.p', 'r.p.d.b', 'r.p.o.t', 'r.p.r.j', 'r.p.s', 'r.r.d', 'r.r.s', 'r.s', 'r.s.v.p', 'r.stvb', 'r.t.d.f', 'r.t.d.h', 'r.t.l', 'r.trim.dr.eur', 'r.v.a', 'r.verkb', 'r.w', 'r.w.d', 'rap.ann.c.a', 'rap.ann.c.c', 'rap.ann.c.e', 'rap.ann.c.s.j', 'rap.ann.ca', 'rap.ann.cass', 'rap.ann.cc', 'rap.ann.ce', 'rap.ann.csj', 'rapp', 'rb', 'rb.kh', 'rb.van kh', 'rdn', 'rdnr', 're.pers', 'rec', 'rec.c.i.j', 'rec.c.j.c.e', 'rec.cij', 'rec.cjce', 'rec.cour eur.d.h', 'rec.gén.enr.not', 'rec.lois decr.arr', 'rechtsk.t', 'rechtspl.zeem', 'rechtspr.arb.br', 'rechtspr.b.f.e', 'rechtspr.bfe', 'rechtspr.soc.r.b.l.n', 'recl.reg', 'rect', 'red', 'reg', 'reg.huiz.bew', 'reg.w', 'registr.w', 'regl', 'regl', 'r.v.k', 'regl.besl', 'regl.onderr', 'regl.r.t', 'rep', 'rep.eur.court h.r', 'rép.fisc', 'rép.not', 'rep.r.j', 'rep.rj', 'req', 'res', 'resp', 'rev', 'rev', 'de dr', 'comp', 'rev', 'trim', 'de dr', 'civ', 'rev', 'trim', 'de dr', 'comm', 'rev.acc.trav', 'rev.adm', 'rev.b.compt', 'rev.b.dr.const', 'rev.b.dr.intern', 'rev.b.séc.soc', 'rev.banc.fin', 'rev.comm', 'rev.cons.prud', 'rev.dr.b', 'rev.dr.commun', 'rev.dr.étr', 'rev.dr.fam', 'rev.dr.intern.comp', 'rev.dr.mil', 'rev.dr.min', 'rev.dr.pén', 'rev.dr.pén.mil', 'rev.dr.rur', 'rev.dr.u.l.b', 'rev.dr.ulb', 'rev.exp', 'rev.faill', 'rev.fisc', 'rev.gd', 'rev.hist.dr', 'rev.i.p.c', 'rev.ipc', 'rev.not.b', 'rev.prat.dr.comm', 'rev.prat.not.b', 'rev.prat.soc', 'rev.rec', 'rev.rw', 'rev.trav', 'rev.trim.d.h', 'rev.trim.dr.fam', 'rev.urb', 'richtl', 'riv.dir.int', 'riv.dir.int."le priv', 'riv.dir.int.priv.proc', 'rk', 'rln', 'roln', 'rom', 'rondz', 'rov', 'rtl', 'rubr', 'ruilv.wet', 'rv.verdr', 'rvkb', 's', 's', 'en s', 's.a', 's.b.n', 's.ct', 's.d', 's.e.c', 's.e.et.o', 's.e.w', 's.exec.rept', 's.hrg', 's.j.b', 's.l', 's.l.e.a', 's.l.n.d', 's.p.a', 's.s', 's.t', 's.t.b', 's.v', 's.v.p', 'samenw', 'sc', 'sch', 'scheidsr.uitspr', 'schepel.besl', 'secr.comm', 'secr.gen', 'sect.soc', 'sess', 'cas', 'sir', 'soc', 'best', 'soc', 'handv', 'soc', 'verz', 'soc.act', 'soc.best', 'soc.kron', 'soc.r', 'soc.sw', 'soc.weg', 'sofi-nr', 'somm', 'somm.ann', 'sp.c.c', 'sr', 'ss', 'st.doc.b.c.n.a.r', 'st.doc.bcnar', 'st.vw', 'stagever', 'stas', 'stat', 'stb', 'stbl', 'stcrt', 'stichting i.v', 'stud.dipl', 'su', 'subs', 'subst', 'succ.w', 'suppl', 'sv', 'sw', 't', 't.a', 't.a.a', 't.a.n', 't.a.p', 't.a.s.n', 't.a.v', 't.a.v.w', 't.aann', 't.acc', 't.agr.r', 't.app', 't.b.b.r', 't.b.h', 't.b.m', 't.b.o', 't.b.p', 't.b.r', 't.b.s', 't.b.v', 't.bankw', 't.belg.not', 't.desk', 't.e.m', 't.e.p', 't.f.r', 't.fam', 't.fin.r', 't.g.r', 't.g.t', 't.g.v', 't.gem', 't.gez', 't.huur', 't.i.n', 't.in b.z', 't.j.k', 't.l.l', 't.l.v', 't.m', 't.m.r', 't.m.w', 't.mil.r', 't.mil.strafr', 't.not', 't.o', 't.o.r.b', 't.o.v', 't.ontv', 't.orde geneesh', 't.p.r', 't.pol', 't.r', 't.r.d.& i', 't.r.g', 't.r.o.s', 't.r.v', 't.s.r', 't.strafr', 't.t', 't.u', 't.v.c', 't.v.g', 't.v.m.r', 't.v.o', 't.v.v', 't.v.v.d.b', 't.v.w', 't.verz', 't.vred', 't.vreemd', 't.w', 't.w.k', 't.w.v', 't.w.v.r', 't.wrr', 't.z', 't.z.t', 't.z.v', 'taalk', 'tar.burg.z', 'td', 'techn', 'telecomm', 'toel', 'toel.st.v.w', 'toep', 'toep.regl', 'tom', 'top', 'trans.b', 'transp.r', 'trav.com.ét.et lég.not', 'trb', 'trib', 'trib.civ', 'trib.gr.inst', 'ts', 'ts', 'best', 'ts', 'verv', 'turnh.rechtsl', 'tvpol', 'tvpr', 'tvrechtsgesch', 'tw', 'u', 'u.a', 'u.a.r', 'u.a.v', 'u.c', 'u.c.c', 'u.g', 'u.p', 'u.s', 'u.s.d.c', 'uitdr', 'uitl.w', 'uitv.besch.div.b', 'uitv.besl', 'uitv.besl', 'succ.w', 'uitv.besl.bel.rv', 'uitv.besl.l.b', 'uitv.reg', 'inv.w', 'uitv.reg.bel.d', 'uitv.reg.afd.verm', 'uitv.reg.lb', 'uitv.reg.succ.w', 'univ', 'univ.verkl', 'v', 'v', 'chr', 'v.& f', 'v.a', 'v.a.v', 'v.bp prot', 'v.c', 'v.chr', 'v.h', 'v.huw.verm', 'v.i', 'v.i.o', 'v.k.a', 'v.m', 'v.o.f', 'v.o.n', 'v.onderh.verpl', 'v.p', 'v.r', 'v.s.o', 'v.t.t', 'v.t.t.t', 'v.tk.t', 'v.toep.r.vert', 'v.v.b', 'v.v.g', 'v.v.t', 'v.v.t.t', 'v.v.tk.t', 'v.w.b', 'v.z.m', 'vb', 'vb.bo', 'vbb', 'vc', 'vd', 'veldw', 'ver.k', 'ver.verg.gem', 'gem.comm', 'verbr', 'verd', 'verdr', 'verdr.v', 'verdrag benel.i.z', 'tek.mod', 'verenw', 'verg', 'verg.fr.gem', 'comm', 'verkl', 'verkl.herz.gw', 'verl', 'deelw', 'vern', 'verord', 'vers.r', 'versch', 'versl.c.s.w', 'versl.csw', 'vert', 'verw', 'verz', 'verz.w', 'verz.wett.besl', 'verz.wett.decr.besl', 'vgl', 'vid', 'vigiles jb', 'viss.w', 'vl.parl', 'vl.r', 'vl.t.gez', 'vl.w.reg', 'vl.w.succ', 'vlg', 'vn', 'vnl', 'vnw', 'vo', 'vo.bl', 'voegw', 'vol', 'volg', 'volt', 'deelw', 'voorl', 'voorz', 'vord.w', 'vorst.d', 'vr', 'en antw', 'vred', 'vrg', 'vnw', 'vrijgrs', 'vs', 'vt', 'vvsr jb', 'vw', 'vz', 'vzngr', 'vzr', 'w', 'w.a', 'w.b.r', 'w.c.h', 'w.conf.huw', 'w.conf.huwelijksb', 'w.consum.kr', 'w.f.r', 'w.g', 'w.gelijke beh', 'w.gew.r', 'w.ident.pl', 'w.just.doc', 'w.kh', 'w.l.r', 'w.l.v', 'w.mil.straf.spr', 'w.n', 'w.not.ambt', 'w.o', 'w.o.d.huurcomm', 'w.o.d.k', 'w.openb.manif', 'w.parl', 'w.r', 'w.reg', 'w.succ', 'w.u.b', 'w.uitv.pl.verord', 'w.v', 'w.v.k', 'w.v.m.s', 'w.v.r', 'w.v.w', 'w.venn', 'wac', 'wd', 'wet a.b', 'wet bel.rv', 'wet c.a.o', 'wet c.o', 'wet div.bel', 'wet ksbel', 'wet l.v', 'wetb', 'n.v.h', 'wgb', 'winkelt.w', 'wisk', 'wka-verkl', 'wnd', 'won.w', 'woningw', 'woonr.w', 'wrr', 'wrr.ber', 'wrsch', 'ws', 'wsch', 'wsr', 'wtvb', 'ww', 'x.d', 'z cont', 'z.a', 'z.g', 'z.i', 'z.j', 'z.o.z', 'z.p', 'z.s.m', 'zesde richtl', 'zg', 'zgn', 'zn', 'znw', 'zr', 'zr', 'ms', 'zr.ms']
+        PREPOSITIVE_ABBREVIATIONS = []
+        NUMBER_ABBREVIATIONS = []

pysbd/lang/english.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+from pysbd.abbreviation_replacer import AbbreviationReplacer
+from pysbd.lang.common import Common, Standard
+
+class English(Common, Standard):
+
+    iso_code = 'en'
+
+    class AbbreviationReplacer(AbbreviationReplacer):
+        SENTENCE_STARTERS = "A Being Did For He How However I In It Millions "\
+            "More She That The There They We What When Where Who Why".split(" ")

pysbd/lang/french.py

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+from pysbd.abbreviation_replacer import AbbreviationReplacer
+from pysbd.lang.common import Common, Standard
+
+class French(Common, Standard):
+
+    iso_code = 'fr'
+
+    class AbbreviationReplacer(AbbreviationReplacer):
+        SENTENCE_STARTERS = []
+
+    class Abbreviation(Standard.Abbreviation):
+        ABBREVIATIONS = ['a.c.n', 'a.m', 'al', 'ann', 'apr', 'art', 'auj', 'av', 'b.p', 'boul', 'c.-à-d', 'c.n', 'c.n.s', 'c.p.i', 'c.q.f.d', 'c.s', 'ca', 'cf', 'ch.-l', 'chap', 'co', 'co', 'contr', 'dir', 'e.g', 'e.v', 'env', 'etc', 'ex', 'fasc', 'fig', 'fr', 'fém', 'hab', 'i.e', 'ibid', 'id', 'inf', 'l.d', 'lib', 'll.aa', 'll.aa.ii', 'll.aa.rr', 'll.aa.ss', 'll.ee', 'll.mm', 'll.mm.ii.rr', 'loc.cit', 'ltd', 'ltd', 'masc', 'mm', 'ms', 'n.b', 'n.d', 'n.d.a', 'n.d.l.r', 'n.d.t', 'n.p.a.i', 'n.s', 'n/réf', 'nn.ss', 'p.c.c', 'p.ex', 'p.j', 'p.s', 'pl', 'pp', 'r.-v', 'r.a.s', 'r.i.p', 'r.p', 's.a', 's.a.i', 's.a.r', 's.a.s', 's.e', 's.m', 's.m.i.r', 's.s', 'sec', 'sect', 'sing', 'sq', 'sqq', 'ss', 'suiv', 'sup', 'suppl', 't.s.v.p', 'tél', 'vb', 'vol', 'vs', 'x.o', 'z.i', 'éd']
+        PREPOSITIVE_ABBREVIATIONS = []
+        NUMBER_ABBREVIATIONS = []

pysbd/lang/greek.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+from pysbd.abbreviation_replacer import AbbreviationReplacer
+from pysbd.lang.common import Common, Standard
+
+class Greek(Common, Standard):
+
+    iso_code = 'el'
+
+    SENTENCE_BOUNDARY_REGEX = r'.*?[\.;!\?]|.*?$'
+    Punctuations = ['.', '!', ';', '?']
+
+    class AbbreviationReplacer(AbbreviationReplacer):
+        SENTENCE_STARTERS = []

pysbd/lang/hindi.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+from pysbd.abbreviation_replacer import AbbreviationReplacer
+from pysbd.lang.common import Common, Standard
+
+class Hindi(Common, Standard):
+
+    iso_code = 'hi'
+
+    SENTENCE_BOUNDARY_REGEX = r'.*?[।\|!\?]|.*?$'
+    Punctuations = ['।', '|', '.', '!', '?']
+
+    class AbbreviationReplacer(AbbreviationReplacer):
+        SENTENCE_STARTERS = []

pysbd/lang/italian.py

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+from pysbd.abbreviation_replacer import AbbreviationReplacer
+from pysbd.lang.common import Common, Standard
+
+class Italian(Common, Standard):
+
+    iso_code = 'it'
+
+    class AbbreviationReplacer(AbbreviationReplacer):
+        SENTENCE_STARTERS = []
+
+    class Abbreviation(Standard.Abbreviation):
+        ABBREVIATIONS = ['1°', 'a.c', 'a.c/a', 'a.cam', 'a.civ', 'a.cor', 'a.d.r', 'a.gov', 'a.mil', 'a.mon', 'a.smv', 'a.v', 'a/a', 'a/c', 'a/i', 'aa', 'aaaa', 'aaal', 'aacst', 'aamct', 'aams', 'aar', 'aato', 'ab', 'abbigl', 'abbrev', 'abc', 'abi', 'abl', 'abm', 'abr', 'abs', 'absp', 'ac', 'acam', 'acb', 'acbi', 'acc', 'accorc', 'accr', 'acd', 'ace', 'acec', 'acep', 'aci', 'acli', 'acp', 'acro', 'acsit', 'actl', 'ad', 'ad.mil', 'ada', 'adap', 'adatt', 'adc', 'add', 'adei', 'adeion', 'adhd', 'adi', 'adisco', 'adj', 'adm', 'adp', 'adr', 'ads', 'adsi', 'adsl', 'adv', 'ae.b', 'aefi', 'aer', 'aerodin', 'aeron', 'afa', 'afc', 'afci', 'affl', 'afi', 'afic', 'afm', 'afp', 'ag', 'agcm', 'agcom', 'age', 'agecs', 'agesci', 'agg', 'agip', 'agis', 'agm', 'ago', 'agr', 'agric', 'agt', 'ai', 'aia', 'aiab', 'aiac', 'aiace', 'aiap', 'aias', 'aiat', 'aib', 'aic', 'aica', 'aicel', 'aici', 'aics', 'aid', 'aida', 'aidaa', 'aidac', 'aidama', 'aidda', 'aidim', 'aido', 'aids', 'aies', 'aif', 'aih', 'aiip', 'aimi', 'aip', 'aipsc', 'airi', 'ais', 'aisa', 'aism', 'aiss', 'aissca', 'aitc', 'aiti', 'aitr', 'aits', 'aka', 'al', 'alai', 'alch', 'alg', 'ali', 'alim', 'all', 'allev', 'allus', 'alp', 'alq', 'alt', 'am', 'ama', 'amaci', 'amag', 'amami', 'amc', 'ammec', 'amn', 'ampas', 'amps', 'an', 'ana', 'anaai', 'anac', 'anaci', 'anad', 'anai', 'anaoo', 'anart', 'anat', 'anat. comp', 'ancci', 'anci', 'ancip', 'ancsa', 'andit', 'anec', 'anee', 'anem', 'anes', 'anffas', 'ani', 'ania', 'anica', 'anie', 'animi', 'anis', 'anisc', 'anm', 'anmfit', 'anmig', 'anmil', 'anmli', 'anms', 'anpa', 'anpas', 'anpci', 'anpe', 'anpi', 'ansi', 'ansv', 'ant', 'anta', 'antifr', 'antlo', 'anton', 'antrop', 'anusca', 'anvi', 'anx', 'ao', 'ap', 'apa', 'apd', 'apea', 'apec', 'apet', 'api', 'apos', 'app', 'app.sc', 'apr', 'aps', 'apt', 'aq', 'ar', 'ar.ind', 'ar.rep', 'arald', 'arame', 'arc', 'arch', 'archeol', 'arci', 'ardsu', 'are', 'arg', 'aritm', 'arpa', 'arpat', 'arred', 'arrt', 'arsia', 'art', 'arti min', 'artig', 'artigl', 'artt', 'as', 'asa', 'asae', 'asc', 'asci', 'ascii', 'ascom', 'ascop', 'asd', 'ase', 'asf', 'asfer', 'asg', 'asic', 'asifa', 'asl', 'asmdc', 'asmi', 'asp', 'aspic', 'aspp', 'assi', 'assic', 'assol', 'asst', 'aster', 'astr', 'astrol', 'astron', 'at', 'ata', 'atb', 'atic', 'atm', 'ats', 'att', 'attrav', 'atv', 'au', 'auc', 'aus', 'auser', 'aut', 'autom', 'av', 'avi', 'avis', 'avo', 'avv', 'avvers', 'awb', 'awdp', 'az', 'azh', 'b.a', 'b2b', 'b2c', 'ba', 'bafta', 'bal', 'ball', 'ban', 'banc', 'bar', 'bart', 'bas', 'bat', 'batt', 'bban', 'bbc', 'bbl', 'bbs', 'bbtc', 'bcc', 'bce', 'bcf', 'bdf', 'bei', 'bep', 'bers', 'bg', 'bi', 'bibl', 'bic', 'bioch', 'biol', 'bl', 'bld', 'bldg', 'blpc', 'bm', 'bmps', 'bmw', 'bn', 'bna', 'bncf', 'bncrm', 'bni', 'bnl', 'bo', 'bot', 'bpl', 'bpm', 'bpn', 'bpr', 'br', 'brd', 'bre', 'bric', 'brig', 'brig.ca', 'brig.gen', 'bros', 'bs', 'bsc', 'bsp', 'bsu', 'bt', 'btc', 'btg', 'btg.l', 'btr', 'bts', 'bu', 'bur', 'bz', 'c.a', 'c.a.p', 'c.c.p', 'c.cost', 'c.d a', 'c.d', 'c.le', 'c.m', 'c.opv', 'c.p', 'c.s', 'c.v', 'c.v.d', 'c/a', 'c/c', 'c/pag', 'ca', 'ca.rep', 'ca.sm', 'ca.sz', 'ca.uf', 'caaf', 'cab', 'cad', 'cae', 'cai', 'cal', 'cam', 'cap', 'capol', 'capt', 'car', 'car.sc', 'carat', 'card', 'cas', 'casaca', 'casd', 'cass.civ', 'cat', 'caus', 'cav', 'cavg', 'cb', 'cbd', 'cbr', 'cbs', 'cc', 'cca', 'ccap', 'ccda', 'ccdp', 'ccee', 'cciaa', 'ccie', 'ccip', 'cciss', 'ccna', 'ccnl', 'ccnp', 'ccpb', 'ccs', 'ccsp', 'cctld', 'cctv', 'ccv', 'cd', 'cda', 'cdma', 'cdo', 'cdpd', 'cdr', 'cds', 'cdw', 'ce', 'ced', 'cee', 'cei', 'cemat', 'cenelec', 'centr', 'cepis', 'ceps', 'cept', 'cerit', 'cese', 'cesis', 'cesvot', 'cet', 'cf', 'cfa', 'cfr', 'cg', 'cgi', 'cgil', 'cgs', 'ch', 'chf', 'chim', 'chim. ind', 'chir', 'ci', 'ci-europa', 'ciber', 'cicae', 'cid', 'cie', 'cif', 'cifej', 'cig', 'cigs', 'cii', 'cilea', 'cilo', 'cim', 'cime', 'cin', 'cinit', 'cio', 'cipe', 'cirm', 'cisal', 'ciscs', 'cisd', 'cisl', 'cism', 'citol', 'cl', 'class', 'cli', 'cm', 'cmdr', 'cme', 'cmo', 'cmr', 'cms', 'cmyk', 'cm²', 'cm³', 'cn', 'cna', 'cnb', 'cnc', 'cnel', 'cngei', 'cni', 'cnipa', 'cnit', 'cnn', 'cnr', 'cns', 'cnt', 'cnvvf', 'co', 'co.ing', 'co.sa', 'cobas', 'coc', 'cod', 'cod. civ', 'cod. deont. not', 'cod. pen', 'cod. proc. civ', 'cod. proc. pen', 'codec', 'coi', 'col', 'colf', 'coll', 'com', 'comdr', 'comm', 'comp', 'compar', 'compl', 'con', 'conai', 'conc', 'concl', 'condiz', 'confetra', 'confitarma', 'confr', 'cong', 'congeav', 'congiunt', 'coni', 'coniug', 'consec', 'consob', 'contab', 'contr', 'coreco', 'corp', 'corr', 'correl', 'corrisp', 'cosap', 'cospe', 'cost', 'costr', 'cpc', 'cpdel', 'cpe', 'cpi', 'cpl', 'cpt', 'cpu', 'cr', 'cral', 'credem', 'crf', 'cri', 'cric', 'cristall', 'crm', 'cro', 'cron', 'crsm', 'crt', 'cs', 'csa', 'csai', 'csc', 'csm', 'csn', 'css', 'ct', 'ctc', 'cti', 'ctr', 'ctsis', 'cuc', 'cud', 'cun', 'cup', 'cusi', 'cvb', 'cvbs', 'cwt', 'cz', 'd', 'd.c', 'd.i.a', 'dab', 'dac', 'dam', 'dams', 'dat', 'dau', 'db', 'dbms', 'dc', 'dca', 'dccc', 'dda', 'ddp', 'ddr', 'ddt', 'dea', 'decoraz', 'dect', 'dek', 'denom', 'deriv', 'derm', 'determ', 'df', 'dfp', 'dg', 'dga', 'dhcp', 'di', 'dia', 'dial', 'dic', 'dicomac', 'dif', 'difett', 'dig. iv', 'digos', 'dimin', 'dimostr', 'din', 'dipart', 'diplom', 'dir', 'dir. amm', 'dir. can', 'dir. civ', 'dir. d. lav', 'dir. giur', 'dir. internaz', 'dir. it', 'dir. pen', 'dir. priv', 'dir. proces', 'dir. pub', 'dir. rom', 'disus', 'diy', 'dl', 'dlf', 'dm', 'dme', 'dmf', 'dmo', 'dmoz', 'dm²', 'dm³', 'dnr', 'dns', 'doa', 'doc', 'docg', 'dom', 'dop', 'dos', 'dott', 'dpa', 'dpi', 'dpl', 'dpof', 'dps', 'dpt', 'dr', 'dra', 'drm', 'drs', 'dry pt', 'ds', 'dslam', 'dspn', 'dss', 'dtc', 'dtmf', 'dtp', 'dts', 'dv', 'dvb', 'dvb-t', 'dvd', 'dvi', 'dwdm', 'e.g', 'e.p.c', 'ead', 'eafrd', 'ean', 'eap', 'easw', 'eb', 'eban', 'ebr', 'ebri', 'ebtn', 'ecc', 'eccl', 'ecdl', 'ecfa', 'ecff', 'ecg', 'ecm', 'econ', 'econ. az', 'econ. dom', 'econ. pol', 'ecpnm', 'ed', 'ed agg', 'edge', 'edi', 'edil', 'edit', 'ef', 'efa', 'efcb', 'efp', 'efsa', 'efta', 'eg', 'egiz', 'egl', 'egr', 'ei', 'eisa', 'elab', 'elettr', 'elettron', 'ellitt', 'emap', 'emas', 'embr', 'emdr', 'emi', 'emr', 'en', 'enaip', 'enal', 'enaoli', 'enapi', 'encat', 'enclic', 'enea', 'enel', 'eni', 'enigm', 'enit', 'enol', 'enpa', 'enpaf', 'enpals', 'enpi', 'enpmf', 'ens', 'entom', 'epd', 'epigr', 'epirbs', 'epl', 'epo', 'ept', 'erc', 'ercom', 'ermes', 'erp', 'es', 'esa', 'escl', 'esist', 'eso', 'esp', 'estens', 'estr. min', 'etacs', 'etf', 'eti', 'etim', 'etn', 'etol', 'eu', 'eufem', 'eufic', 'eula', 'eva®', 'f.a', 'f.b', 'f.m', 'f.p', 'fa', 'fabi', 'fac', 'facl', 'facs', 'fad', 'fai', 'faile', 'failp', 'failpa', 'faisa', 'falcri', 'fam', 'famar', 'fans', 'fao', 'fapav', 'faq', 'farm', 'fasi', 'fasib', 'fatt', 'fbe', 'fbi', 'fc', 'fco', 'fcp', 'fcr', 'fcu', 'fdi', 'fe', 'feaog', 'feaosc', 'feb', 'fedic', 'fema', 'feoga', 'ferr', 'fesco', 'fesr', 'fess', 'fg', 'fi', 'fiaf', 'fiaip', 'fiais', 'fialtel', 'fiap', 'fiapf', 'fiat', 'fiavet', 'fic', 'ficc', 'fice', 'fidal', 'fidam', 'fidapa', 'fieg', 'fifa', 'fifo', 'fig', 'figc', 'figs', 'filat', 'filcams', 'file', 'filol', 'filos', 'fim', 'fima', 'fimmg', 'fin', 'finco', 'fio', 'fioto', 'fipe', 'fipresci', 'fis', 'fisar', 'fisc', 'fisg', 'fisiol', 'fisiopatol', 'fistel', 'fit', 'fita', 'fitav', 'fits', 'fiv', 'fivet', 'fivl', 'flo', 'flpd', 'fluid pt', 'fm', 'fmcg', 'fmi', 'fmth', 'fnas', 'fnomceo', 'fnsi', 'fob', 'fod', 'folcl', 'fon', 'fop', 'fotogr', 'fp', 'fpc', 'fpld', 'fr', 'fra', 'fs', 'fsc', 'fse', 'fsf', 'fsfi', 'fsh', 'ft', 'ftase', 'ftbcc', 'fte', 'ftp', 'fts', 'ft²', 'ft³', 'fuaav', 'fut', 'fv', 'fvg', 'g.fv', 'g.u', 'g.u.el', 'gal', 'gats', 'gatt', 'gb', 'gc', 'gccc', 'gco', 'gcost', 'gd', 'gdd', 'gdf', 'gdi', 'gdo', 'gdp', 'ge', 'gea', 'gel', 'gen', 'geneal', 'geod', 'geofis', 'geogr', 'geogr. antr', 'geogr. fis', 'geol', 'geom', 'gep', 'germ', 'gescal', 'gg', 'ggv', 'gi', 'gia', 'gides', 'gift', 'gio', 'giorn', 'gis', 'gisma', 'gismo', 'giu', 'gm', 'gmdss', 'gme', 'gmo', 'go', 'gov', 'gp', 'gpl', 'gprs', 'gps', 'gr', 'gr.sel.spec', 'gr.sel.tr', 'gr.sqd', 'gra', 'gram', 'grano', 'grd', 'grtn', 'grv', 'gsa', 'gsm', 'gsm-r', 'gsr', 'gtld', 'gu', 'guce', 'gui', 'gus', 'ha', 'haart', 'haccp', 'hba', 'hcg', 'hcrp', 'hd-dvd', 'hdcp', 'hdi', 'hdml', 'hdtv', 'hepa', 'hfpa', 'hg', 'hifi', 'hiperlan', 'hiv', 'hm', 'hmld', 'hon', 'hosp', 'hpv', 'hr', 'hrh', 'hrm', 'hrt', 'html', 'http', 'hvac', 'hz', 'i.e', 'i.g.m', 'iana', 'iasb', 'iasc', 'iass', 'iat', 'iata', 'iatse', 'iau', 'iban', 'ibid', 'ibm', 'icann', 'icao', 'icbi', 'iccu', 'ice', 'icf', 'ici', 'icm', 'icom', 'icon', 'ics', 'icsi', 'icstis', 'ict', 'icta', 'id', 'iden', 'idl', 'idraul', 'iec', 'iedm', 'ieee', 'ietf', 'ifat', 'ifel', 'ifla', 'ifrs', 'ifto', 'ifts', 'ig', 'igm', 'igmp', 'igp', 'iims', 'iipp', 'ilm', 'ilo', 'ilor', 'ils', 'im', 'imaie', 'imap', 'imc', 'imdb', 'imei', 'imi', 'imms', 'imo', 'imp', 'imper', 'imperf', 'impers', 'imq', 'ims', 'imsi', 'in', 'inail', 'inca', 'incb', 'inci', 'ind', 'ind. agr', 'ind. alim', 'ind. cart', 'ind. chim', 'ind. cuoio', 'ind. estratt', 'ind. graf', 'ind. mecc', 'ind. tess', 'indecl', 'indef', 'indeterm', 'indire', 'inea', 'inf', 'infea', 'infm', 'inform', 'ing', 'ingl', 'inmarsat', 'inpdai', 'inpdap', 'inpgi', 'inps', 'inr', 'inran', 'ins', 'insp', 'int', 'inter', 'intr', 'invar', 'invim', 'in²', 'in³', 'ioma', 'iosco', 'ip', 'ipab', 'ipasvi', 'ipi', 'ippc', 'ips', 'iptv', 'iq', 'ira', 'irap', 'ircc', 'ircs', 'irda', 'iref', 'ires', 'iron', 'irpef', 'irpeg', 'irpet', 'irreg', 'is', 'isae', 'isbd', 'isbn', 'isc', 'isdn', 'isee', 'isef', 'isfol', 'isg', 'isi', 'isia', 'ism', 'ismea', 'isnart', 'iso', 'isp', 'ispearmi', 'ispel', 'ispescuole', 'ispesl', 'ispo', 'ispro', 'iss', 'issn', 'istat', 'istol', 'isvap', 'it', 'iti', 'itt', 'ittiol', 'itu', 'iud', 'iugr', 'iulm', 'iva', 'iveco', 'ivg', 'ivr', 'ivs', 'iyhp', 'j', 'jal', 'jit', 'jr', 'jv', 'k', 'kb', 'kee', 'kg', 'kkk', 'klm', 'km', 'km/h', 'kmph', 'kmq', 'km²', 'kr', 'kw', 'kwh', 'l', 'l\'ing', 'l.n', 'l\'avv', 'la', 'lag', 'lan', 'lanc', 'larn', 'laser', 'lat', 'lav', 'lav. femm', 'lav. pubbl', 'laz', 'lb', 'lc', 'lcca', 'lcd', 'le', 'led', 'lett', 'lh', 'li', 'liaf', 'lib', 'lic', 'lic.ord', 'lic.strd', 'licd', 'lice', 'lida', 'lidci', 'liff', 'lifo', 'lig', 'liit', 'lila', 'lilt', 'linfa', 'ling', 'lipu', 'lis', 'lisaac', 'lism', 'lit', 'litab', 'lnp', 'lo', 'loc', 'loc. div', 'lolo', 'lom', 'long', 'lp', 'lrm', 'lrms', 'lsi', 'lsu', 'lt', 'ltd', 'lu', 'lug', 'luiss', 'lun', 'lwt', 'lww', 'm.a', 'm.b', 'm.o', 'm/s', 'ma', 'mac', 'macch', 'mag', 'magg.(maj)', 'magg.gen.(maj.gen.)', 'mai', 'maj', 'mar', 'mar.a', 'mar.ca', 'mar.ord', 'marc', 'mat', 'mater', 'max', 'mb', 'mbac', 'mc', 'mcl', 'mcpc', 'mcs', 'md', 'mdf', 'mdp', 'me', 'mec', 'mecc', 'med', 'mediev', 'mef', 'mer', 'merc', 'merid', 'mesa', 'messrs', 'metall', 'meteor', 'metr', 'metrol', 'mg', 'mgc', 'mgm', 'mi', 'mibac', 'mica', 'microb', 'mifed', 'miglio nautico', 'miglio nautico per ora', 'miglio nautico²', 'miglio²', 'mil', 'mile', 'miles/h', 'milesph', 'min', 'miner', 'mips', 'miptv', 'mit', 'mitol', 'miur', 'ml', 'mlle', 'mls', 'mm', 'mme', 'mms', 'mm²', 'mn', 'mnp', 'mo', 'mod', 'mol', 'mons', 'morf', 'mos', 'mpaa', 'mpd', 'mpeg', 'mpi', 'mps', 'mq', 'mr', 'mrs', 'ms', 'msgr', 'mss', 'mt', 'mto', 'murst', 'mus', 'mvds', 'mws', 'm²', 'm³', 'n.a', 'n.b', 'na', 'naa', 'nafta', 'napt', 'nars', 'nasa', 'nat', 'natas', 'nato', 'nb', 'nba', 'nbc', 'ncts', 'nd', 'nda', 'nde', 'ndr', 'ndt', 'ne', 'ned', 'neg', 'neol', 'netpac', 'neur', 'news!', 'ngcc', 'nhmf', 'nlcc', 'nmr', 'no', 'nodo', 'nom', 'nos', 'nov', 'novissdi', 'npi', 'nr', 'nt', 'nta', 'nts', 'ntsc', 'nu', 'nuct', 'numism', 'nwt', 'nyc', 'nz', 'o.m.i', 'oai-pmh', 'oav', 'oc', 'occ', 'occult', 'oci', 'ocr', 'ocse', 'oculist', 'od', 'odg', 'odp', 'oecd', 'oem', 'ofdm', 'oft', 'og', 'ogg', 'ogi', 'ogm', 'ohim', 'oic', 'oics', 'olaf', 'oland', 'ole', 'oled', 'omi', 'oms', 'on', 'ong', 'onig', 'onlus', 'onomat', 'onpi', 'onu', 'op', 'opac', 'opec', 'opord', 'opsosa', 'or', 'ord', 'ord. scol', 'ore', 'oref', 'orient', 'ornit', 'orogr', 'orp', 'ort', 'os', 'osa', 'osas', 'osd', 'ot', 'ote', 'ott', 'oz', 'p', 'p.a', 'p.c', 'p.c.c', 'p.es', 'p.f', 'p.m', 'p.r', 'p.s', 'p.t', 'p.v', 'pa', 'pac', 'pag./p', 'pagg./pp', 'pai', 'pal', 'paleobot', 'paleogr', 'paleont', 'paleozool', 'paletn', 'pamr', 'pan', 'papir', 'par', 'parapsicol', 'part', 'partic', 'pass', 'pat', 'patol', 'pb', 'pc', 'pci', 'pcm', 'pcmcia', 'pcs', 'pcss', 'pct', 'pd', 'pda', 'pdf', 'pdl', 'pds', 'pe', 'pec', 'ped', 'pedag', 'peg', 'pegg', 'per.ind', 'pers', 'pert', 'pesq', 'pet', 'petr', 'petrogr', 'pfc', 'pg', 'pga', 'pgp', 'pgut', 'ph', 'php', 'pi', 'pics', 'pie', 'pif', 'pii', 'pil', 'pime', 'pin', 'pine', 'pip', 'pir', 'pit', 'pitt', 'piuss', 'pkcs', 'pki', 'pko', 'pl', 'pli', 'plr', 'pm', 'pma', 'pmi', 'pmr', 'pn', 'pnf', 'pnl', 'po', 'poet', 'pof', 'pol', 'pop', 'popitt', 'popol', 'port', 'pos', 'poss', 'post', 'pots', 'pp', 'ppa', 'ppc', 'ppga', 'ppp', 'pps', 'pptt', 'ppv', 'pr', 'pra', 'praa', 'pref', 'preist', 'prep', 'pres', 'pret', 'prg', 'pri', 'priv', 'pro.civ', 'prof', 'pron', 'pronom', 'propr', 'prov', 'prs', 'prtl', 'prusst', 'ps', 'pse', 'psi', 'psicoan', 'psicol', 'pso', 'psp', 'pstn', 'pt', 'ptc', 'pti', 'ptsd', 'ptt', 'pu', 'pug', 'puk', 'put', 'pv', 'pvb', 'pvc', 'pvt', 'pz', 'qb', 'qcs', 'qfd', 'qg', 'qi', 'qlco', 'qlcu', 'qos', 'qualif', 'r-lan', 'r.s', 'ra', 'racc', 'radar', 'radc', 'radiotecn', 'raee', 'raf', 'rag', 'raid', 'ram', 'rar', 'ras', 'rass. avv. stato', 'rc', 'rca', 'rcdp', 'rcs', 'rdc', 'rdco', 'rdf', 'rdi', 'rdp', 'rds', 'rdt', 're', 'rea', 'recipr', 'recl', 'reg', 'region', 'rel', 'rem', 'rep', 'reps', 'res', 'retor', 'rev', 'rfi', 'rfid', 'rg', 'rgb', 'rgc', 'rge', 'rgi', 'rgi bdp', 'rgpt', 'rgt', 'ri', 'riaa', 'riaj', 'riba', 'ric', 'rid', 'rif', 'rifl', 'rina', 'rip', 'ris', 'rit', 'ritts', 'rm', 'rmn', 'rn', 'ro', 'roa', 'roc', 'roi', 'rom', 'roro', 'rov', 'rp', 'rpm', 'rr', 'rrf', 'rs', 'rsc', 'rspp', 'rss', 'rsu', 'rsvp', 'rt', 'rtdpc', 'rtg', 'rtn', 'rtp', 'rttt', 'rvm', 's-dab', 's.a', 's.b.f', 's.n.c', 's.p.a', 's.p.m', 's.r.l', 's.ten', 's.v', 's/m', 'sa', 'sab', 'saca', 'sace', 'sact', 'sad', 'sag', 'sahm', 'sai', 'saisa', 'sam', 'san', 'sanas', 'sape', 'sar', 'sars', 'sart', 'sas', 'sbaf', 'sbas', 'sbn', 'sc', 'sca.sm', 'scherz', 'scien', 'scn', 'scsi', 'scuba', 'scult', 'scut', 'sdds', 'sdiaf', 'sds', 'sdsl', 'se', 'seat', 'sebc', 'sec', 'seca', 'secam', 'secc', 'see', 'seg', 'segg', 'segredifesa', 'sem', 'sempo', 'sen', 'sens', 'seo', 'serg', 'serg.magg.(sgm)', 'serg.magg.ca', 'set', 'sfc', 'sfis', 'sfx', 'sg', 'sga', 'sgc', 'sgg', 'sgml', 'sgt', 'si', 'si@lt', 'sia', 'siae', 'siaic', 'siap', 'sias', 'sic', 'sicav', 'sid', 'sido', 'sie', 'sif', 'sig', 'sig.na', 'sig.ra', 'sige', 'sigg', 'sigill', 'sigo', 'siia', 'simb', 'simbdea', 'simg', 'simo', 'sin', 'sinalv', 'sing', 'sins', 'sinu', 'siocmf', 'siog', 'sioi', 'siommms', 'siot', 'sip', 'sipem', 'sips', 'sirf', 'sirm', 'sis', 'sisde', 'sismi', 'sissa', 'sit', 'siulp', 'siusa', 'sla', 'sldn', 'slm', 'slr', 'sm', 'sma', 'smau', 'smd', 'sme', 'smes', 'smm', 'smpt', 'sms', 'sn', 'snad', 'snai', 'snc', 'sncci', 'sncf', 'sngci', 'snit', 'so', 'soc', 'sociol', 'sogg', 'soho', 'soi', 'sol', 'somipar', 'somm', 'sonar', 'sp', 'spa', 'spe', 'spett', 'spi', 'spm', 'spot', 'spp', 'spreg', 'sq', 'sqd', 'sr', 'srd', 'srl', 'srr', 'ss', 'ssi', 'ssn', 'ssr', 'sss', 'st', 'st. d. arte', 'st. d. dir', 'st. d. filos', 'st. d. rel', 'stat', 'stg', 'stp', 'stw', 'su', 'suap', 'suem', 'suff', 'sup', 'superl', 'supt', 'surg', 'surl', 'susm', 'sut', 'suv', 'sv', 'svga', 'swics', 'swift', 'swot', 'sxga', 'sz', 't-dab', 't.sg', 'ta', 'taa', 'tac', 'tacan', 'tacs', 'taeg', 'tai', 'tan', 'tar', 'targa', 'tav', 'tb', 'tbt', 'tci', 'tcp', 'tcp/ip', 'tcsm', 'tdm', 'tdma', 'te', 'tecn', 'tecnol', 'ted', 'tel', 'telecom', 'temp', 'ten.(lt)', 'ten.col.(ltc)', 'ten.gen', 'teol', 'term', 'tesa', 'tese', 'tesol', 'tess', 'tet', 'tetra', 'tfr', 'tft', 'tfts', 'tgv', 'thx', 'tim', 'tipogr', 'tir', 'tit', 'tld', 'tm', 'tmc', 'tn', 'to', 'toefl', 'ton', 'top', 'topog', 'tos', 'tosap', 'tosc', 'tp', 'tpl', 'tr', 'trad', 'tramat', 'trasp', 'ts', 'tso', 'tuir', 'tuld', 'tv', 'twa', 'twain', 'u.ad', 'u.s', 'ucai', 'ucca', 'ucei', 'ucina', 'uclaf', 'ucoi', 'ucoii', 'ucsi', 'ud', 'udc', 'udi', 'udp', 'ue', 'uefa', 'uemri', 'ufo', 'ugc', 'uhci', 'uhf', 'uht', 'uibm', 'uic', 'uicc', 'uiga', 'uil', 'uilps', 'uisp', 'uits', 'uk', 'ul', 'ull', 'uma', 'umb', 'ummc', 'umss', 'umts', 'unac', 'unar', 'unasp', 'uncem', 'unctad', 'undp', 'unefa', 'unep', 'unesco', 'ungh', 'unhcr', 'uni', 'unicef', 'unitec', 'unpredep', 'unsa', 'upa', 'upc', 'urar', 'urban', 'url', 'urp', 'urss', 'usa', 'usb', 'usfi', 'usga', 'usl', 'usp', 'uspi', 'ussr', 'utap', 'v', 'v.brig', 'v.cte', 'v.m', 'v.p', 'v.r', 'v.s', 'va', 'vab', 'vaio', 'val', 'vas', 'vb', 'vbr', 'vc', 'vcc', 'vcr', 'vda', 've', 'ven', 'ves', 'vesa', 'veter', 'vezz', 'vfb', 'vfp', 'vfx', 'vga', 'vhf', 'vhs', 'vi', 'via', 'vip', 'vis', 'vn', 'vo', 'voc', 'voip', 'vol', 'volg', 'voll', 'vor', 'vpdn', 'vpn', 'vr', 'vs', 'vsp', 'vt', 'vtc', 'vts', 'vtt', 'vv', 'vvf', 'wai', 'wais', 'wan', 'wap', 'wasp', 'wc', 'wcdma', 'wcm', 'wga', 'wi-fi', 'wipo', 'wisp', 'wll', 'wml', 'wms', 'worm', 'wp', 'wpan', 'wssn', 'wto', 'wwan', 'wwf', 'www', 'wygiwys', 'xl', 'xml', 'xs', 'xxl', 'xxs', 'yaf', 'yb', 'yci', 'yd', 'yd²', 'yd³', 'ymca', 'zat', 'zb', 'zcs', 'zdf', 'zdg', 'zift', 'zool', 'zoot', 'ztc', 'ztl', '°c', '°f', '°n', '°ra', '°ré', 'µg']
+        PREPOSITIVE_ABBREVIATIONS = ['a.c', 'acc', 'adj', 'adm', 'adv', 'all', 'amn', 'arch', 'asst', 'avv', 'banc', 'bart', 'bcc', 'bldg', 'brig', 'bros', 'c.a', 'c.a.p', 'c.c.p', 'c.m', 'c.p', 'c.p', 'c.s', 'c.v', 'capt', 'cc', 'cmdr', 'co', 'col', 'comdr', 'con', 'corp', 'corr', 'cpl', 'dir', 'dott', 'dott', 'dr', 'dr', 'drs', 'e.p.c', 'ecc', 'egr', 'ens', 'es', 'fatt', 'gen', 'geom', 'gg', 'gov', 'hon', 'hosp', 'hr', 'id', 'ing', 'insp', 'int', "l'avv", "l'ing", 'lett', 'lt', 'maj', 'messrs', 'mlle', 'mm', 'mme', 'mo', 'mons', 'mr', 'mr', 'mrs', 'mrs', 'ms', 'ms', 'msgr', 'n.b', 'ogg', 'on', 'op', 'ord', 'p.c', 'p.c.c', 'p.es', 'p.f', 'p.r', 'p.s', 'p.t', 'p.v', 'pfc', 'ph', 'post', 'pp', 'prof', 'psicol', 'pvt', 'racc', 'rag', 'rep', 'reps', 'res', 'rev', 'ric', 'rif', 'rp', 'rsvp', 'rt', 's.a', 's.b.f', 's.n.c', 's.p.a', 's.p.m', 's.r.l', 'seg', 'sen', 'sens', 'sfc', 'sgg', 'sgt', 'sig', 'sigg', 'soc', 'spett', 'sr', 'ss', 'st', 'supt', 'surg', 'tel', 'u.s', 'v.p', 'v.r', 'v.s']
+        NUMBER_ABBREVIATIONS = ['art', 'no', 'nos', 'nr', 'pp']

pysbd/lang/japanese.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+import re
+from pysbd.abbreviation_replacer import AbbreviationReplacer
+from pysbd.between_punctuation import BetweenPunctuation
+from pysbd.lang.common import Common, Standard
+from pysbd.punctuation_replacer import replace_punctuation
+from pysbd.cleaner import Cleaner
+from pysbd.utils import Text, Rule
+
+class Japanese(Common, Standard):
+
+    iso_code = 'ja'
+
+    class Cleaner(Cleaner):
+
+        def __init__(self, text, lang, doc_type=None):
+            super().__init__(text, lang)
+
+        def clean(self):
+            self.remove_newline_in_middle_of_word()
+            return self.text
+
+        def remove_newline_in_middle_of_word(self):
+            NewLineInMiddleOfWordRule = Rule(r'(?<=の)\n(?=\S)', '')
+            self.text = Text(self.text).apply(NewLineInMiddleOfWordRule)
+
+    class AbbreviationReplacer(AbbreviationReplacer):
+        SENTENCE_STARTERS = []
+
+    class BetweenPunctuation(BetweenPunctuation):
+
+        def __init__(self, text):
+            super().__init__(text)
+
+        def replace(self):
+            self.sub_punctuation_between_quotes_and_parens()
+            return self.text
+
+        def sub_punctuation_between_parens_ja(self):
+            BETWEEN_PARENS_JA_REGEX = r'（(?=(?P<tmp>[^（）]+|\\{2}|\\.)*)(?P=tmp)）'
+            self.text = re.sub(BETWEEN_PARENS_JA_REGEX, replace_punctuation,
+                      self.text)
+
+        def sub_punctuation_between_quotes_ja(self):
+            BETWEEN_QUOTE_JA_REGEX = r'「(?=(?P<tmp>[^「」]+|\\{2}|\\.)*)(?P=tmp)」'
+            self.text = re.sub(BETWEEN_QUOTE_JA_REGEX, replace_punctuation,
+                      self.text)
+
+        def sub_punctuation_between_quotes_and_parens(self):
+            self.sub_punctuation_between_parens_ja()
+            self.sub_punctuation_between_quotes_ja()

pysbd/lang/kazakh.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+from pysbd.abbreviation_replacer import AbbreviationReplacer
+from pysbd.lang.common import Common, Standard
+from pysbd.processor import Processor
+from pysbd.utils import Text, Rule
+
+
+class Kazakh(Common, Standard):
+
+    iso_code = 'kk'
+
+    # Handling Cyrillic characters in re module
+    # https://stackoverflow.com/a/10982308/5462100
+    MULTI_PERIOD_ABBREVIATION_REGEX = r'\b[\u0400-\u0500]+(?:\.\s?[\u0400-\u0500])+[.]|b[a-z](?:\.[a-z])+[.]'
+
+    class Processor(Processor):
+
+        def __init__(self, text, lang, char_span=False):
+            super().__init__(text, lang, char_span)
+
+        def between_punctuation(self, txt):
+            txt = self.between_punctuation_processor(txt).replace()
+            # Rubular: http://rubular.com/r/WRWy56Z5zp
+            QuestionMarkFollowedByDashLowercaseRule = Rule(r'(?<=)\?(?=\s*[-—]\s*)', '&ᓷ&')
+            # Rubular: http://rubular.com/r/lixxP7puSa
+            ExclamationMarkFollowedByDashLowercaseRule = Rule(r'(?<=)!(?=\s*[-—]\s*)', '&ᓴ&')
+
+            txt = Text(txt).apply(QuestionMarkFollowedByDashLowercaseRule,
+                                  ExclamationMarkFollowedByDashLowercaseRule)
+            return txt
+
+    class Abbreviation(Standard.Abbreviation):
+        ABBREVIATIONS = ['afp', 'anp', 'atp', 'bae', 'bg', 'bp', 'cam', 'cctv', 'cd', 'cez', 'cgi', 'cnpc', 'farc', 'fbi', 'eiti', 'epo', 'er', 'gp', 'gps', 'has', 'hiv', 'hrh', 'http', 'icu', 'idf', 'imd', 'ime', 'icu', 'idf', 'ip', 'iso', 'kaz', 'kpo', 'kpa', 'kz', 'kz', 'mri', 'nasa', 'nba', 'nbc', 'nds', 'ohl', 'omlt', 'ppm', 'pda', 'pkk', 'psm', 'psp', 'raf', 'rss', 'rtl', 'sas', 'sme', 'sms', 'tnt', 'udf', 'uefa', 'usb', 'utc', 'x', 'zdf', 'әқбк', 'әқбк', 'аақ', 'авг.', 'aбб', 'аек', 'ак', 'ақ', 'акцион.', 'акср', 'ақш', 'англ', 'аөсшк', 'апр', 'м.', 'а.', 'р.', 'ғ.', 'апр.', 'аум.', 'ацат', 'әч', 'т. б.', 'б. з. б.', 'б. з. б.', 'б. з. д.', 'б. з. д.', 'биікт.', 'б. т.', 'биол.', 'биохим', 'бө', 'б. э. д.', 'бта', 'бұұ', 'вич', 'всоонл', 'геогр.', 'геол.', 'гленкор', 'гэс', 'қк', 'км', 'г', 'млн', 'млрд', 'т', 'ғ. с.', 'ғ.', 'қ.', 'ғ.', 'дек.', 'днқ', 'дсұ', 'еақк', 'еқыұ', 'ембімұнайгаз', 'ео', 'еуразэқ', 'еуроодақ', 'еұу', 'ж.', 'ж.', 'жж.', 'жоо', 'жіө', 'жсдп', 'жшс', 'іім', 'инта', 'исаф', 'камаз', 'кгб', 'кеу', 'кг', 'км²', 'км²', 'км³', 'км³', 'кимеп', 'кср', 'ксро', 'кокп', 'кхдр', 'қазатомпром', 'қазкср', 'қазұу', 'қазмұнайгаз', 'қазпошта', 'қазтаг', 'қазұу', 'қкп', 'қмдб', 'қр', 'қхр', 'лат.', 'м²', 'м²', 'м³', 'м³', 'магатэ', 'май.', 'максам', 'мб', 'мвт', 'мемл', 'м', 'мсоп', 'мтк', 'мыс.', 'наса', 'нато', 'нквд', 'нояб.', 'обл.', 'огпу', 'окт.', 'оңт.', 'опек', 'оеб', 'өзенмұнайгаз', 'өф', 'пәк', 'пед.', 'ркфср', 'рнқ', 'рсфср', 'рф', 'свс', 'сву', 'сду', 'сес', 'сент.', 'см', 'снпс', 'солт.', 'солт.', 'сооно', 'ссро', 'сср', 'ссср', 'ссс', 'сэс', 'дк', 'т. б.', 'т', 'тв', 'тереңд.', 'тех.', 'тжқ', 'тмд', 'төм.', 'трлн', 'тр', 'т.', 'и.', 'м.', 'с.', 'ш.', 'т.', 'т. с. с.', 'тэц', 'уаз', 'уефа', 'еқыұ', 'ұқк', 'ұқшұ', 'февр.', 'фққ', 'фсб', 'хим.', 'хқко', 'шұар', 'шыұ', 'экон.', 'экспо', 'цтп', 'цас', 'янв.', 'dvd', 'жкт', 'ққс', 'км', 'ацат', 'юнеско', 'ббс', 'mgm', 'жск', 'зоо', 'бсн', 'өұқ', 'оар', 'боак', 'эөкк', 'хтқо', 'әөк', 'жэк', 'хдо', 'спбму', 'аф', 'сбд', 'амт', 'гсдп', 'гсбп', 'эыдұ', 'нұсжп', 'шыұ', 'жтсх', 'хдп', 'эқк', 'фкққ', 'пиқ', 'өгк', 'мбф', 'маж', 'кота', 'тж', 'ук', 'обб', 'сбл', 'жхл', 'кмс', 'бмтрк', 'жққ', 'бхооо', 'мқо', 'ржмб', 'гулаг', 'жко', 'еэы', 'еаэы', 'кхдр', 'рфкп', 'рлдп', 'хвқ', 'мр', 'мт', 'кту', 'ртж', 'тим', 'мемдум', 'ксро', 'т.с.с', 'с.ш.', 'ш.б.', 'б.б.', 'руб', 'мин', 'акад.', 'ғ.', 'мм', 'мм.']
+        PREPOSITIVE_ABBREVIATIONS = []
+        NUMBER_ABBREVIATIONS = []
+
+    class AbbreviationReplacer(AbbreviationReplacer):
+
+        SENTENCE_STARTERS = []
+
+        def __init__(self, text, lang):
+            super().__init__(text, lang)
+
+        def replace(self):
+            SingleUpperCaseCyrillicLetterAtStartOfLineRule = Rule(r'(?<=^[А-ЯЁ])\.(?=\s)', '∯')
+            SingleUpperCaseCyrillicLetterRule = Rule(r'(?<=\s[А-ЯЁ])\.(?=\s)', '∯')
+            self.text = Text(self.text).apply(SingleUpperCaseCyrillicLetterAtStartOfLineRule,
+                                              SingleUpperCaseCyrillicLetterRule)
+            self.replace_multi_period_abbreviations()
+            return self.text

pysbd/lang/marathi.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+# Grammer rules from https://gopract.com/Pages/Marathi-Grammar-Viramchinah.aspx
+from pysbd.abbreviation_replacer import AbbreviationReplacer
+from pysbd.lang.common import Common, Standard
+
+class Marathi(Common, Standard):
+
+    iso_code = 'mr'
+
+    SENTENCE_BOUNDARY_REGEX = r'.*?[.!?]|.*?$'
+    Punctuations = ['.', '!', '?']
+
+    class AbbreviationReplacer(AbbreviationReplacer):
+        SENTENCE_STARTERS = []

pysbd/lang/persian.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+import re
+
+from pysbd.abbreviation_replacer import AbbreviationReplacer
+from pysbd.lang.common import Common, Standard
+from pysbd.utils import Rule
+
+class Persian(Common, Standard):
+
+    iso_code = 'fa'
+
+    Punctuations = ['?', '!', ':', '.', '؟']
+    SENTENCE_BOUNDARY_REGEX = r'.*?[:\.!\?؟]|.*?\Z|.*?$'
+
+    # Rubular: http://rubular.com/r/RX5HpdDIyv
+    ReplaceColonBetweenNumbersRule = Rule(r'(?<=\d):(?=\d)', '♭')
+
+    # Rubular: http://rubular.com/r/kPRgApNHUg
+    ReplaceNonSentenceBoundaryCommaRule = Rule(r'،(?=\s\S+،)', '♬')
+
+    class AbbreviationReplacer(AbbreviationReplacer):
+
+        SENTENCE_STARTERS = []
+
+        def __init__(self, text, lang):
+            super().__init__(text, lang)
+
+        def scan_for_replacements(self, txt, am, index, character_array):
+            txt = re.sub('(?<={0})\.'.format(am), '∯', txt)
+            return txt

pysbd/lang/polish.py

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+from pysbd.abbreviation_replacer import AbbreviationReplacer
+from pysbd.lang.common import Common, Standard
+
+class Polish(Common, Standard):
+
+    iso_code = 'pl'
+
+    class AbbreviationReplacer(AbbreviationReplacer):
+        SENTENCE_STARTERS = []
+
+    class Abbreviation(Standard.Abbreviation):
+        ABBREVIATIONS = ['ags', 'alb', 'ang', 'aor', 'awest', 'bałt', 'bojkow', 'bret', 'brus', 'bsł', 'bułg', 'c.b.d.o', 'c.b.d.u', 'celt', 'chorw', 'cs', 'czakaw', 'czerw', 'czes', 'dłuż', 'dniem', 'dor', 'dubrow', 'duń', 'ekaw', 'fiń', 'franc', 'gal', 'germ', 'głuż', 'gniem', 'goc', 'gr', 'grudz', 'hebr', 'het', 'hol', 'I cont', 'ie', 'ikaw', 'irań', 'irl', 'islandz', 'itd', 'itd.', 'itp', 'jekaw', 'kajkaw', 'kasz', 'kirg', 'kwiec', 'łac', 'lip', 'listop', 'lit', 'łot', 'lp', 'maced', 'mar', 'młpol', 'moraw', 'n.e', 'nb.', 'ngr', 'niem', 'nord', 'norw', 'np', 'np.', 'ok.', 'orm', 'oset', 'osk', 'p.n', 'p.n.e', 'p.o', 'pazdz', 'pers', 'pie', 'pod red.', 'podhal', 'pol', 'połab', 'port', 'prekm', 'pskow', 'psł', 'R cont', 'rez', 'rom', 'rozdz.', 'rum', 'rus', 'rys.', 'sas', 'sch', 'scs', 'serb', 'sierp', 'śl', 'sła', 'słe', 'słi', 'słow', 'sp. z o.o', 'śrdniem', 'śrgniem', 'śrirl', 'stbułg', 'stind', 'stpol', 'stpr', 'str.', 'strus', 'stwniem', 'stycz', 'sztokaw', 'szwedz', 't.', 'tj.', 'tłum.', 'toch', 'tur', 'tzn', 'ukr', 'ul', 'umbr', 'wed', 'węg', 'wlkpol', 'włos', 'wrzes', 'wyd.', 'zakarp']
+        PREPOSITIVE_ABBREVIATIONS = []
+        NUMBER_ABBREVIATIONS = []

pysbd/lang/russian.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+import re
+
+from pysbd.abbreviation_replacer import AbbreviationReplacer
+from pysbd.lang.common import Common, Standard
+
+class Russian(Common, Standard):
+
+    iso_code = 'ru'
+
+    class Abbreviation(Standard.Abbreviation):
+        ABBREVIATIONS = ["y", "y.e", "а", "авт", "адм.-терр", "акад", "в", "вв", "вкз", "вост.-европ", "г", "гг", "гос", "гр", "д", "деп", "дисс", "дол", "долл", "ежедн", "ж", "жен", "з", "зап", "зап.-европ", "заруб", "и", "ин", "иностр", "инст", "к", "канд", "кв", "кг", "куб", "л", "л.h", "л.н", "м", "мин", "моск", "муж", "н", "нед", "о", "п", "пгт", "пер", "пп", "пр", "просп", "проф", "р", "руб", "с", "сек", "см", "спб", "стр", "т", "тел", "тов", "тт", "тыс", "у", "у.е", "ул", "ф", "ч"]
+        PREPOSITIVE_ABBREVIATIONS = []
+        NUMBER_ABBREVIATIONS = []
+
+    class AbbreviationReplacer(AbbreviationReplacer):
+
+        SENTENCE_STARTERS = []
+
+        def __init__(self, text, lang):
+            super().__init__(text, lang)
+
+        def replace_period_of_abbr(self, txt, abbr):
+            txt = re.sub(r'(?<=\s{abbr})\.'.format(abbr=abbr.strip()), '∯', txt)
+            txt = re.sub(r'(?<=\A{abbr})\.'.format(abbr=abbr.strip()), '∯', txt)
+            txt = re.sub(r'(?<=^{abbr})\.'.format(abbr=abbr.strip()), '∯', txt)
+            return txt

pysbd/lang/slovak.py

@@ -0,0 +1,111 @@
+# -*- coding: utf-8 -*-
+import re
+from pysbd.abbreviation_replacer import AbbreviationReplacer
+from pysbd.between_punctuation import BetweenPunctuation
+from pysbd.lang.common import Common, Standard
+from pysbd.processor import Processor
+from pysbd.utils import Text
+from pysbd.punctuation_replacer import replace_punctuation
+from pysbd.lists_item_replacer import ListItemReplacer
+
+
+class Slovak(Common, Standard):
+
+    iso_code = 'sk'
+
+    class ListItemReplacer(ListItemReplacer):
+
+        def add_line_break(self):
+            # We've found alphabetical lists are causing a lot of problems with abbreviations
+            # with multiple periods and spaces, such as 'Company name s. r. o.'. Disabling
+            # alphabetical list parsing seems like a reasonable tradeoff.
+
+            # self.format_alphabetical_lists()
+            self.format_roman_numeral_lists()
+            self.format_numbered_list_with_periods()
+            self.format_numbered_list_with_parens()
+            return self.text
+
+    class AbbreviationReplacer(AbbreviationReplacer):
+        SENTENCE_STARTERS = []
+
+        def replace_period_of_abbr(self, txt, abbr):
+            # This is a very simple version of the original function, which makes sure
+            # all of the periods in the abbreviation get replaced, not only the last one.
+            # In Slovak language we use a lot of abbreviations like 'Company Name s. r. o.', so it
+            # is important to handle this properly.
+
+            abbr_new = abbr.replace(".", "∯") + "∯"
+            txt = txt.replace(abbr + ".", abbr_new)
+            return txt
+
+    class Abbreviation(Standard.Abbreviation):
+        ABBREVIATIONS = ['č', 'no', 'nr', 's. r. o', 'ing', 'p', 'a. d', 'o. k', 'pol. pr', 'a. s. a. p', 'p. n. l', 'red', 'o.k', 'a.d', 'm.o', 'pol.pr', 'a.s.a.p', 'p.n.l', 'pp', 'sl', 'corp', 'plgr', 'tz', 'rtg', 'o.c.p', 'o. c. p', 'c.k', 'c. k', 'n.a', 'n. a', 'a.m', 'a. m', 'vz', 'i.b', 'i. b', 'ú.p.v.o', 'ú. p. v. o', 'bros', 'rsdr', 'doc', 'tu', 'ods', 'n.w.a', 'n. w. a', 'nár', 'pedg', 'paeddr', 'rndr', 'naprk', 'a.g.p', 'a. g. p', 'prof', 'pr', 'a.v', 'a. v', 'por', 'mvdr', 'nešp', 'u.s', 'u. s', 'kt', 'vyd', 'e.t', 'e. t', 'al', 'll.m', 'll. m', 'o.f.i', 'o. f. i', 'mr', 'apod', 'súkr', 'stred', 's.e.g', 's. e. g', 'sr', 'tvz', 'ind', 'var', 'etc', 'atd', 'n.o', 'n. o', 's.a', 's. a', 'např', 'a.i.i', 'a. i. i', 'a.k.a', 'a. k. a', 'konkr', 'čsl', 'odd', 'ltd', 't.z', 't. z', 'o.z', 'o. z', 'obv', 'obr', 'pok', 'tel', 'št', 'skr', 'phdr', 'xx', 'š.p', 'š. p', 'ph.d', 'ph. d', 'm.n.m', 'm. n. m', 'zz', 'roz', 'atď.', 'ev', 'v.sp', 'v. sp', 'drsc', 'mudr', 't.č', 't. č', 'el', 'os', 'co', 'r.o', 'r. o', 'str', 'p.a', 'p. a', 'zdravot', 'prek', 'gen', 'viď', 'dr', 'cca', 'p.s', 'p. s', 'zák', 'slov', 'arm', 'inc', 'max', 'd.c', 'k.o', 'a. r. k', 'd. c', 'k. o', 'a. r. k', 'soc', 'bc', 'zs', 'akad', 'sz', 'pozn', 'tr', 'nám', 'kol', 'csc', 'ul', 'sp', 'o.i', 'jr', 'zb', 'sv', 'tj', 'čs', 'tzn', 'príp', 'iv', 'hl', 'st', 'pod', 'vi', 'tis', 'stor', 'rozh', 'mld', 'atď', 'mgr', 'a.s', 'a. s', 'phd', 'z.z', 'z. z', 'judr', 'ing', 'hod', 'vs', 'písm', 's.r.o', 'min', 'ml', 'iii', 't.j', 't. j', 'spol', 'mil', 'ii', 'napr', 'resp', 'tzv']
+        PREPOSITIVE_ABBREVIATIONS = ['st', 'p', 'dr', 'mudr', 'judr', 'ing', 'mgr', 'bc', 'drsc', 'doc', 'prof']
+        NUMBER_ABBREVIATIONS = ['č', 'no', 'nr']
+
+    class BetweenPunctuation(BetweenPunctuation):
+        # Rubular: https://rubular.com/r/rImWbaYFtHHtf4
+        BETWEEN_SLOVAK_DOUBLE_QUOTES_REGEX = r'„(?>[^“\\]+|\\{2}|\\.)*“'
+        BETWEEN_SLOVAK_DOUBLE_QUOTES_REGEX_2 = r'\„(?=(?P<tmp>[^“\\]+|\\{2}|\\.)*)(?P=tmp)\“'
+
+        def sub_punctuation_between_slovak_double_quotes(self, txt):
+            return re.sub(self.BETWEEN_SLOVAK_DOUBLE_QUOTES_REGEX_2, replace_punctuation, txt)
+
+        def sub_punctuation_between_quotes_and_parens(self, txt):
+            txt = self.sub_punctuation_between_single_quotes(txt)
+            txt = self.sub_punctuation_between_single_quote_slanted(txt)
+            txt = self.sub_punctuation_between_double_quotes(txt)
+            txt = self.sub_punctuation_between_square_brackets(txt)
+            txt = self.sub_punctuation_between_parens(txt)
+            txt = self.sub_punctuation_between_quotes_arrow(txt)
+            txt = self.sub_punctuation_between_em_dashes(txt)
+            txt = self.sub_punctuation_between_quotes_slanted(txt)
+            txt = self.sub_punctuation_between_slovak_double_quotes(txt)
+            return txt
+
+    class Processor(Processor):
+
+        def __init__(self, text, lang, char_span=False):
+            super().__init__(text, lang, char_span)
+
+        def process(self):
+            if not self.text:
+                return self.text
+            self.text = self.text.replace('\n', '\r')
+
+            # Here we use language specific ListItemReplacer:
+            li = self.lang.ListItemReplacer(self.text)
+            self.text = li.add_line_break()
+
+            self.replace_abbreviations()
+            self.replace_numbers()
+            self.replace_continuous_punctuation()
+            self.replace_periods_before_numeric_references()
+            self.text = Text(self.text).apply(
+                self.lang.Abbreviation.WithMultiplePeriodsAndEmailRule,
+                self.lang.GeoLocationRule, self.lang.FileFormatRule)
+            postprocessed_sents = self.split_into_segments()
+            return postprocessed_sents
+
+        def replace_numbers(self):
+            self.text = Text(self.text).apply(*self.lang.Numbers.All)
+            self.replace_period_in_slovak_dates()
+            self.replace_period_in_ordinal_numerals()
+            self.replace_period_in_roman_numerals()
+            return self.text
+
+        def replace_period_in_ordinal_numerals(self):
+            # Rubular: https://rubular.com/r/0HkmvzMGTqgWs6
+            self.text = re.sub(r'(?<=\d)\.(?=\s*[a-z]+)', '∯', self.text)
+
+        def replace_period_in_roman_numerals(self):
+            # Rubular: https://rubular.com/r/XlzTIi7aBRThSl
+            self.text = re.sub(r'((\s+[VXI]+)|(^[VXI]+))(\.)(?=\s+)', r'\1∯', self.text, re.IGNORECASE)
+
+        def replace_period_in_slovak_dates(self):
+            MONTHS = ['Január', 'Február', 'Marec', 'Apríl', 'Máj', 'Jún', 'Júl', 'August', 'September', 'Október', 'November', 'December',
+                      'Januára', 'Februára', 'Marca', 'Apríla', 'Mája', 'Júna', 'Júla', 'Augusta', 'Septembra', 'Októbra', 'Novembra', 'Decembra']
+            for month in MONTHS:
+                # Rubular: https://rubular.com/r/dGLZqsbjcdJvCd
+                self.text = re.sub(r'(?<=\d)\.(?=\s*{month})'.format(month=month), '∯', self.text)

pysbd/lang/spanish.py

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+from pysbd.abbreviation_replacer import AbbreviationReplacer
+from pysbd.lang.common import Common, Standard
+
+class Spanish(Common, Standard):
+
+    iso_code = 'es'
+
+    class AbbreviationReplacer(AbbreviationReplacer):
+        SENTENCE_STARTERS = []
+
+    class Abbreviation(Standard.Abbreviation):
+        ABBREVIATIONS = ['a.c', 'a/c', 'abr', 'adj', 'admón', 'afmo', 'ago', 'almte', 'ap', 'apdo', 'arq', 'art', 'atte', 'av', 'avda', 'bco', 'bibl', 'bs. as', 'c', 'c.f', 'c.g', 'c/c', 'c/u', 'cap', 'cc.aa', 'cdad', 'cm', 'co', 'cra', 'cta', 'cv', 'd.e.p', 'da', 'dcha', 'dcho', 'dep', 'dic', 'dicc', 'dir', 'dn', 'doc', 'dom', 'dpto', 'dr', 'dra', 'dto', 'ee', 'ej', 'en', 'entlo', 'esq', 'etc', 'excmo', 'ext', 'f.c', 'fca', 'fdo', 'febr', 'ff. aa', 'ff.cc', 'fig', 'fil', 'fra', 'g.p', 'g/p', 'gob', 'gr', 'gral', 'grs', 'hnos', 'hs', 'igl', 'iltre', 'imp', 'impr', 'impto', 'incl', 'ing', 'inst', 'izdo', 'izq', 'izqdo', 'j.c', 'jue', 'jul', 'jun', 'kg', 'km', 'lcdo', 'ldo', 'let', 'lic', 'ltd', 'lun', 'mar', 'may', 'mg', 'min', 'mié', 'mm', 'máx', 'mín', 'mt', 'n. del t', 'n.b', 'no', 'nov', 'ntra. sra', 'núm', 'oct', 'p', 'p.a', 'p.d', 'p.ej', 'p.v.p', 'párrf', 'ppal', 'prev', 'prof', 'prov', 'ptas', 'pts', 'pza', 'pág', 'págs', 'párr', 'q.e.g.e', 'q.e.p.d', 'q.e.s.m', 'reg', 'rep', 'rr. hh', 'rte', 's', 's. a', 's.a.r', 's.e', 's.l', 's.r.c', 's.r.l', 's.s.s', 's/n', 'sdad', 'seg', 'sept', 'sig', 'sr', 'sra', 'sres', 'srta', 'sta', 'sto', 'sáb', 't.v.e', 'tamb', 'tel', 'tfno', 'ud', 'uu', 'uds', 'univ', 'v.b', 'v.e', 'vd', 'vds', 'vid', 'vie', 'vol', 'vs', 'vto', 'a', 'aero', 'ambi', 'an', 'anfi', 'ante', 'anti', 'archi', 'arci', 'auto', 'bi', 'bien', 'bis', 'co', 'com', 'con', 'contra', 'crio', 'cuadri', 'cuasi', 'cuatri', 'de', 'deci', 'des', 'di', 'dis', 'dr', 'ecto', 'en', 'endo', 'entre', 'epi', 'equi', 'ex', 'extra', 'geo', 'hemi', 'hetero', 'hiper', 'hipo', 'homo', 'i', 'im', 'in', 'infra', 'inter', 'intra', 'iso', 'lic', 'macro', 'mega', 'micro', 'mini', 'mono', 'multi', 'neo', 'omni', 'para', 'pen', 'ph', 'ph.d', 'pluri', 'poli', 'pos', 'post', 'pre', 'pro', 'pseudo', 're', 'retro', 'semi', 'seudo', 'sobre', 'sub', 'super', 'supra', 'trans', 'tras', 'tri', 'ulter', 'ultra', 'un', 'uni', 'vice', 'yuxta']
+        PREPOSITIVE_ABBREVIATIONS = ['a', 'aero', 'ambi', 'an', 'anfi', 'ante', 'anti', 'archi', 'arci', 'auto', 'bi', 'bien', 'bis', 'co', 'com', 'con', 'contra', 'crio', 'cuadri', 'cuasi', 'cuatri', 'de', 'deci', 'des', 'di', 'dis', 'dr', 'ecto', 'ee', 'en', 'endo', 'entre', 'epi', 'equi', 'ex', 'extra', 'geo', 'hemi', 'hetero', 'hiper', 'hipo', 'homo', 'i', 'im', 'in', 'infra', 'inter', 'intra', 'iso', 'lic', 'macro', 'mega', 'micro', 'mini', 'mono', 'mt', 'multi', 'neo', 'omni', 'para', 'pen', 'ph', 'ph.d', 'pluri', 'poli', 'pos', 'post', 'pre', 'pro', 'prof', 'pseudo', 're', 'retro', 'semi', 'seudo', 'sobre', 'sub', 'super', 'supra', 'sra', 'srta', 'trans', 'tras', 'tri', 'ulter', 'ultra', 'un', 'uni', 'vice', 'yuxta']
+        NUMBER_ABBREVIATIONS = ['cra', 'ext', 'no', 'nos', 'p', 'pp', 'tel']

pysbd/lang/urdu.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+from pysbd.abbreviation_replacer import AbbreviationReplacer
+from pysbd.lang.common import Common, Standard
+
+class Urdu(Common, Standard):
+
+    iso_code = 'ur'
+
+    SENTENCE_BOUNDARY_REGEX = r'.*?[۔؟!\?]|.*?$'
+    Punctuations = ['?', '!', '۔', '؟']
+
+    class AbbreviationReplacer(AbbreviationReplacer):
+        SENTENCE_STARTERS = []

pysbd/languages.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+from pysbd.lang.english import English
+from pysbd.lang.hindi import Hindi
+from pysbd.lang.marathi import Marathi
+from pysbd.lang.chinese import Chinese
+from pysbd.lang.spanish import Spanish
+from pysbd.lang.amharic import Amharic
+from pysbd.lang.arabic import Arabic
+from pysbd.lang.armenian import Armenian
+from pysbd.lang.bulgarian import Bulgarian
+from pysbd.lang.urdu import Urdu
+from pysbd.lang.russian import Russian
+from pysbd.lang.polish import Polish
+from pysbd.lang.persian import Persian
+from pysbd.lang.dutch import Dutch
+from pysbd.lang.danish import Danish
+from pysbd.lang.french import French
+from pysbd.lang.burmese import Burmese
+from pysbd.lang.greek import Greek
+from pysbd.lang.italian import Italian
+from pysbd.lang.japanese import Japanese
+from pysbd.lang.deutsch import Deutsch
+from pysbd.lang.kazakh import Kazakh
+from pysbd.lang.slovak import Slovak
+from pysbd.lang.armenian import Armenian
+
+LANGUAGE_CODES = {
+    'en': English,
+    'hi': Hindi,
+    'mr': Marathi,
+    'zh': Chinese,
+    'es': Spanish,
+    'am': Amharic,
+    'ar': Arabic,
+    'hy': Armenian,
+    'bg': Bulgarian,
+    'ur': Urdu,
+    'ru': Russian,
+    'pl': Polish,
+    'fa': Persian,
+    'nl': Dutch,
+    'da': Danish,
+    'fr': French,
+    'my': Burmese,
+    'el': Greek,
+    'it': Italian,
+    'ja': Japanese,
+    'de': Deutsch,
+    'kk': Kazakh,
+    'sk': Slovak,
+    'hy': Armenian
+}
+
+
+class Language(object):
+
+    def __init__(self, code):
+        self.code = code
+
+    @classmethod
+    def get_language_code(cls, code):
+        try:
+            return LANGUAGE_CODES[code]
+        except KeyError:
+            raise ValueError("Provide valid language ID i.e. ISO code. "
+                "Available codes are : {}".format(set(LANGUAGE_CODES.keys())))

pysbd/lists_item_replacer.py

@@ -0,0 +1,240 @@
+# -*- coding: utf-8 -*-
+import string
+import re
+from pysbd.utils import Rule, Text
+from functools import partial
+
+
+class ListItemReplacer(object):
+
+    ROMAN_NUMERALS = "i ii iii iv v vi vii viii ix x xi xii xiii xiv x xi xii xiii xv xvi xvii xviii xix xx".split(' ')
+    LATIN_NUMERALS = list(string.ascii_lowercase)
+
+    # Rubular: http://rubular.com/r/XcpaJKH0sz
+    ALPHABETICAL_LIST_WITH_PERIODS = r'(?<=^)[a-z](?=\.)|(?<=\A)[a-z](?=\.)|(?<=\s)[a-z](?=\.)'
+
+    # Rubular: http://rubular.com/r/Gu5rQapywf
+    # TODO: Make sure below regex call is case-insensitive
+    ALPHABETICAL_LIST_WITH_PARENS = r'(?<=\()[a-z]+(?=\))|(?<=^)[a-z]+(?=\))|(?<=\A)[a-z]+(?=\))|(?<=\s)[a-z]+(?=\))'
+
+    # (pattern, replacement)
+    SubstituteListPeriodRule = Rule('♨', '∯')
+    ListMarkerRule = Rule('☝', '')
+
+    # Rubular: http://rubular.com/r/Wv4qLdoPx7
+    # https://regex101.com/r/62YBlv/1
+    SpaceBetweenListItemsFirstRule = Rule(r'(?<=\S\S)\s(?=\S\s*\d+♨)', "\r")
+
+    # Rubular: http://rubular.com/r/AizHXC6HxK
+    # https://regex101.com/r/62YBlv/2
+    SpaceBetweenListItemsSecondRule = Rule(r'(?<=\S\S)\s(?=\d{1,2}♨)', "\r")
+
+    # Rubular: http://rubular.com/r/GE5q6yID2j
+    # https://regex101.com/r/62YBlv/3
+    SpaceBetweenListItemsThirdRule = Rule(r'(?<=\S\S)\s(?=\d{1,2}☝)', "\r")
+
+    NUMBERED_LIST_REGEX_1 = r'\s\d{1,2}(?=\.\s)|^\d{1,2}(?=\.\s)|\s\d{1,2}(?=\.\))|^\d{1,2}(?=\.\))|(?<=\s\-)\d{1,2}(?=\.\s)|(?<=^\-)\d{1,2}(?=\.\s)|(?<=\s\⁃)\d{1,2}(?=\.\s)|(?<=^\⁃)\d{1,2}(?=\.\s)|(?<=s\-)\d{1,2}(?=\.\))|(?<=^\-)\d{1,2}(?=\.\))|(?<=\s\⁃)\d{1,2}(?=\.\))|(?<=^\⁃)\d{1,2}(?=\.\))'
+    # 1. abcd
+    # 2. xyz
+    NUMBERED_LIST_REGEX_2 = r'(?<=\s)\d{1,2}\.(?=\s)|^\d{1,2}\.(?=\s)|(?<=\s)\d{1,2}\.(?=\))|^\d{1,2}\.(?=\))|(?<=\s\-)\d{1,2}\.(?=\s)|(?<=^\-)\d{1,2}\.(?=\s)|(?<=\s\⁃)\d{1,2}\.(?=\s)|(?<=^\⁃)\d{1,2}\.(?=\s)|(?<=\s\-)\d{1,2}\.(?=\))|(?<=^\-)\d{1,2}\.(?=\))|(?<=\s\⁃)\d{1,2}\.(?=\))|(?<=^\⁃)\d{1,2}\.(?=\))'
+    # 1) abcd
+    # 2) xyz
+    NUMBERED_LIST_PARENS_REGEX = r'\d{1,2}(?=\)\s)'
+
+    # Rubular: http://rubular.com/r/NsNFSqrNvJ
+    # TODO: Make sure below regex call is case-insensitive
+    EXTRACT_ALPHABETICAL_LIST_LETTERS_REGEX = r'\([a-z]+(?=\))|(?<=^)[a-z]+(?=\))|(?<=\A)[a-z]+(?=\))|(?<=\s)[a-z]+(?=\))'
+
+    # Rubular: http://rubular.com/r/wMpnVedEIb
+    # TODO: Make sure below regex call is case-insensitive
+    ALPHABETICAL_LIST_LETTERS_AND_PERIODS_REGEX = r'(?<=^)[a-z]\.|(?<=\A)[a-z]\.|(?<=\s)[a-z]\.'
+
+    # Rubular: http://rubular.com/r/GcnmQt4a3I
+    ROMAN_NUMERALS_IN_PARENTHESES = r'\(((?=[mdclxvi])m*(c[md]|d?c*)(x[cl]|l?x*)(i[xv]|v?i*))\)(?=\s[A-Z])'
+
+    def __init__(self, text):
+        self.text = text
+
+    def add_line_break(self):
+        self.format_alphabetical_lists()
+        self.format_roman_numeral_lists()
+        self.format_numbered_list_with_periods()
+        self.format_numbered_list_with_parens()
+        return self.text
+
+    def replace_parens(self):
+        text = re.sub(self.ROMAN_NUMERALS_IN_PARENTHESES,
+                      r'&✂&\1&⌬&', self.text)
+        return text
+
+    def format_numbered_list_with_parens(self):
+        self.replace_parens_in_numbered_list()
+        self.add_line_breaks_for_numbered_list_with_parens()
+        self.text = Text(self.text).apply(self.ListMarkerRule)
+
+    def replace_periods_in_numbered_list(self):
+        self.scan_lists(self.NUMBERED_LIST_REGEX_1, self.NUMBERED_LIST_REGEX_2,
+                        '♨', strip=True)
+
+    def format_numbered_list_with_periods(self):
+        self.replace_periods_in_numbered_list()
+        self.add_line_breaks_for_numbered_list_with_periods()
+        self.text = Text(self.text).apply(self.SubstituteListPeriodRule)
+
+    def format_alphabetical_lists(self):
+        self.txt = self.add_line_breaks_for_alphabetical_list_with_periods(
+            roman_numeral=False)
+        self.txt = self.add_line_breaks_for_alphabetical_list_with_parens(
+            roman_numeral=False)
+        return self.txt
+
+    def format_roman_numeral_lists(self):
+        self.txt = self.add_line_breaks_for_alphabetical_list_with_periods(
+            roman_numeral=True)
+        self.txt = self.add_line_breaks_for_alphabetical_list_with_parens(
+            roman_numeral=True)
+        return self.txt
+
+    def add_line_breaks_for_alphabetical_list_with_periods(
+            self, roman_numeral=False):
+        txt = self.iterate_alphabet_array(
+            self.ALPHABETICAL_LIST_WITH_PERIODS,
+            roman_numeral=roman_numeral)
+        return txt
+
+    def add_line_breaks_for_alphabetical_list_with_parens(self, roman_numeral=False):
+        txt = self.iterate_alphabet_array(
+            self.ALPHABETICAL_LIST_WITH_PARENS,
+            parens=True,
+            roman_numeral=roman_numeral)
+        return txt
+
+    def scan_lists(self, regex1, regex2, replacement, strip=False):
+        list_array = re.findall(regex1, self.text)
+        list_array = list(map(int, list_array))
+        for ind, item in enumerate(list_array):
+            # to avoid IndexError
+            # ruby returns nil if index is out of range
+            if (ind < len(list_array) - 1 and item + 1 == list_array[ind + 1]):
+                self.substitute_found_list_items(regex2, item, strip, replacement)
+            elif ind > 0:
+                if (((item - 1) == list_array[ind - 1]) or
+                    ((item == 0) and (list_array[ind - 1] == 9)) or
+                    ((item == 9) and (list_array[ind - 1] == 0))):
+                    self.substitute_found_list_items(regex2, item, strip, replacement)
+
+    def substitute_found_list_items(self, regex, each, strip, replacement):
+
+        def replace_item(match, val=None, strip=False, repl='♨'):
+            match = match.group()
+            if strip:
+                match = str(match).strip()
+            chomped_match = match if len(match) == 1 else match.strip('.])')
+            if str(each) == chomped_match:
+                return "{}{}".format(each, replacement)
+            else:
+                return str(match)
+
+        self.text = re.sub(regex, partial(replace_item, val=each,
+                           strip=strip, repl=replacement), self.text)
+
+    def add_line_breaks_for_numbered_list_with_periods(self):
+        if ('♨' in self.text) and (not re.search(
+                '♨.+(\n|\r).+♨', self.text)) and (not re.search(
+                    r'for\s\d{1,2}♨\s[a-z]', self.text)):
+            self.text = Text(self.text).apply(self.SpaceBetweenListItemsFirstRule,
+                                    self.SpaceBetweenListItemsSecondRule)
+
+    def replace_parens_in_numbered_list(self):
+        self.scan_lists(
+            self.NUMBERED_LIST_PARENS_REGEX, self.NUMBERED_LIST_PARENS_REGEX, '☝')
+        self.scan_lists(self.NUMBERED_LIST_PARENS_REGEX, self.NUMBERED_LIST_PARENS_REGEX, '☝')
+
+    def add_line_breaks_for_numbered_list_with_parens(self):
+        if '☝' in self.text and not re.search("☝.+\n.+☝|☝.+\r.+☝", self.text):
+            self.text = Text(self.text).apply(
+                self.SpaceBetweenListItemsThirdRule)
+
+    def replace_alphabet_list(self, a):
+        """
+        Input: 'a. ffegnog b. fgegkl c.'
+        Output: \ra∯ ffegnog \rb∯ fgegkl \rc∯
+        """
+
+        def replace_letter_period(match, val=None):
+            match = match.group()
+            match_wo_period = match.strip('.')
+            if match_wo_period == val:
+                return '\r{}∯'.format(match_wo_period)
+            else:
+                return match
+
+        txt = re.sub(self.ALPHABETICAL_LIST_LETTERS_AND_PERIODS_REGEX,
+                     partial(replace_letter_period, val=a),
+                     self.text, flags=re.IGNORECASE)
+        return txt
+
+    def replace_alphabet_list_parens(self, a):
+        """
+        Input: "a) ffegnog (b) fgegkl c)"
+        Output: "\ra) ffegnog \r&✂&b) fgegkl \rc)"
+        """
+
+        def replace_alphabet_paren(match, val=None):
+            match = match.group()
+            if '(' in match:
+                match_wo_paren = match.strip('(')
+                if match_wo_paren == val:
+                    return '\r&✂&{}'.format(match_wo_paren)
+                else:
+                    return match
+            else:
+                if match == val:
+                    return '\r{}'.format(match)
+                else:
+                    return match
+
+        # Make it cases-insensitive
+        txt = re.sub(self.EXTRACT_ALPHABETICAL_LIST_LETTERS_REGEX,
+                     partial(replace_alphabet_paren, val=a),
+                     self.text, flags=re.IGNORECASE)
+        return txt
+
+    def replace_correct_alphabet_list(self, a, parens):
+        if parens:
+            a = self.replace_alphabet_list_parens(a)
+        else:
+            a = self.replace_alphabet_list(a)
+        return a
+
+    def last_array_item_replacement(self, a, i, alphabet, list_array, parens):
+        if (len(alphabet) == 0) & (len(list_array) == 0) or (
+                list_array[i - 1] not in alphabet) or (a not in alphabet):
+            return self.text
+        if abs(alphabet.index(list_array[i - 1]) - alphabet.index(a)) != 1:
+            return self.text
+        result = self.replace_correct_alphabet_list(a, parens)
+        return result
+
+    def other_items_replacement(self, a, i, alphabet, list_array, parens):
+        if (len(alphabet) == 0) & (len(list_array) == 0) or (
+                list_array[i - 1] not in alphabet) or (a not in alphabet) or (
+                    list_array[i + 1] not in alphabet):
+            return self.text
+        if alphabet.index(list_array[i + 1]) - alphabet.index(a) != 1 and \
+                abs(alphabet.index(list_array[i - 1]) - alphabet.index(a)) != 1:
+            return self.text
+        result = self.replace_correct_alphabet_list(a, parens)
+        return result
+
+    def iterate_alphabet_array(self, regex, parens=False, roman_numeral=False):
+        list_array = re.findall(regex, self.text)
+        alphabet = self.ROMAN_NUMERALS if roman_numeral else self.LATIN_NUMERALS
+        list_array = [i for i in list_array if i in alphabet]
+        for ind, each in enumerate(list_array):
+            if ind == len(list_array) - 1:
+                self.text = self.last_array_item_replacement(each, ind, alphabet, list_array, parens)
+            else:
+                self.text = self.other_items_replacement(
+                    each, ind, alphabet, list_array, parens)
+        return self.text

pysbd/processor.py

@@ -0,0 +1,210 @@
+# -*- coding: utf-8 -*-
+import re
+from pysbd.utils import Text
+from pysbd.lists_item_replacer import ListItemReplacer
+from pysbd.exclamation_words import ExclamationWords
+from pysbd.between_punctuation import BetweenPunctuation
+from pysbd.abbreviation_replacer import AbbreviationReplacer
+
+class Processor(object):
+
+    def __init__(self, text, lang, char_span=False):
+        """Process a text - do pre and post processing - to get proper sentences
+
+        Parameters
+        ----------
+        text : str
+            Original text
+        language : object
+            Language module
+        char_span : bool, optional
+            Get start & end character offsets of each sentences
+            within original text, by default False
+        """
+        self.text = text
+        self.lang = lang
+        self.char_span = char_span
+
+    def process(self):
+        if not self.text:
+            return self.text
+        self.text = self.text.replace('\n', '\r')
+        li = ListItemReplacer(self.text)
+        self.text = li.add_line_break()
+        self.replace_abbreviations()
+        self.replace_numbers()
+        self.replace_continuous_punctuation()
+        self.replace_periods_before_numeric_references()
+        self.text = Text(self.text).apply(
+            self.lang.Abbreviation.WithMultiplePeriodsAndEmailRule,
+            self.lang.GeoLocationRule, self.lang.FileFormatRule)
+        postprocessed_sents = self.split_into_segments()
+        return postprocessed_sents
+
+    def rm_none_flatten(self, sents):
+        """Remove None values and unpack list of list sents
+
+        Parameters
+        ----------
+        sents : list
+            list of sentences
+
+        Returns
+        -------
+        list
+            unpacked and None removed list of sents
+        """
+        sents = list(filter(None, sents))
+        if not any(isinstance(s, list) for s in sents):
+            return sents
+        new_sents = []
+        for sent in sents:
+            if isinstance(sent, list):
+                for s in sent:
+                    new_sents.append(s)
+            else:
+                new_sents.append(sent)
+        return new_sents
+
+    def split_into_segments(self):
+        self.check_for_parens_between_quotes()
+        sents = self.text.split('\r')
+        # remove empty and none values
+        sents = self.rm_none_flatten(sents)
+        sents = [
+            Text(s).apply(self.lang.SingleNewLineRule, *self.lang.EllipsisRules.All)
+            for s in sents
+        ]
+        
+        # # THESE LINES ARE NOT PRESENT IN THE ORIGINAL CODE --> ONLY USE FOR HYW
+        # sents = [self.post_process_segments(s) for s in sents]
+        # sents = self.rm_none_flatten(sents)
+        
+
+        sents = [self.check_for_punctuation(s) for s in sents]
+        # flatten list of list of sentences
+        sents = self.rm_none_flatten(sents)
+        postprocessed_sents = []
+        for sent in sents:
+            sent = Text(sent).apply(*self.lang.SubSymbolsRules.All)
+            post_process_sent = self.post_process_segments(sent)
+            if post_process_sent and isinstance(post_process_sent, str):
+                postprocessed_sents.append(post_process_sent)
+            elif isinstance(post_process_sent, list):
+                for pps in post_process_sent:
+                    postprocessed_sents.append(pps)
+        postprocessed_sents = [Text(ns).apply(self.lang.SubSingleQuoteRule)
+                               for ns in postprocessed_sents]
+        return postprocessed_sents
+
+    def post_process_segments(self, txt):
+        if len(txt) > 2 and re.search(r'\A[a-zA-Z]*\Z', txt):
+            return txt
+
+        # below condition present in pragmatic segmenter
+        # dont know significance of it yet.
+        # if self.consecutive_underscore(txt) or len(txt) < 2:
+        #     return txt
+
+        if re.match(r'\t', txt):
+            pass
+
+        # TODO:
+        # Decide on keeping or removing Standard.ExtraWhiteSpaceRule
+        # removed to retain original text spans
+        # txt = Text(txt).apply(*ReinsertEllipsisRules.All,
+        #                       Standard.ExtraWhiteSpaceRule)
+        txt = Text(txt).apply(*self.lang.ReinsertEllipsisRules.All)
+        if re.search(self.lang.QUOTATION_AT_END_OF_SENTENCE_REGEX, txt):
+            txt = re.split(
+                self.lang.SPLIT_SPACE_QUOTATION_AT_END_OF_SENTENCE_REGEX, txt)
+            return txt
+        else:
+            txt = txt.replace('\n', '')
+            return txt.strip()
+
+    def check_for_parens_between_quotes(self):
+        def paren_replace(match):
+            match = match.group()
+            sub1 = re.sub(r'\s(?=\()', '\r', match)
+            sub2 = re.sub(r'(?<=\))\s', '\r', sub1)
+            return sub2
+        self.text = re.sub(self.lang.PARENS_BETWEEN_DOUBLE_QUOTES_REGEX,
+                      paren_replace, self.text)
+
+    def replace_continuous_punctuation(self):
+        def continuous_puncs_replace(match):
+            match = match.group()
+            sub1 = re.sub(re.escape('!'), '&ᓴ&', match)
+            sub2 = re.sub(re.escape('?'), '&ᓷ&', sub1)
+            return sub2
+        self.text = re.sub(self.lang.CONTINUOUS_PUNCTUATION_REGEX,
+                        continuous_puncs_replace, self.text)
+
+    def replace_periods_before_numeric_references(self):
+         # https://github.com/diasks2/pragmatic_segmenter/commit/d9ec1a352aff92b91e2e572c30bb9561eb42c703
+        self.text = re.sub(self.lang.NUMBERED_REFERENCE_REGEX,
+                      r"∯\2\r\7", self.text)
+
+    def consecutive_underscore(self, txt):
+        # Rubular: http://rubular.com/r/fTF2Ff3WBL
+        txt = re.sub(r'_{3,}', '', txt)
+        return len(txt) == 0
+
+    def check_for_punctuation(self, txt):
+        if any(p in txt for p in self.lang.Punctuations):
+            sents = self.process_text(txt)
+            return sents
+        else:
+            # NOTE: next steps of check_for_punctuation will unpack this list
+            return [txt]
+
+    def process_text(self, txt):
+        if txt[-1] not in self.lang.Punctuations:
+            txt += 'ȸ'
+        txt = ExclamationWords.apply_rules(txt)
+        txt = self.between_punctuation(txt)
+        # handle text having only doublepunctuations
+        if not re.match(self.lang.DoublePunctuationRules.DoublePunctuation, txt):
+            txt = Text(txt).apply(*self.lang.DoublePunctuationRules.All)
+        txt = Text(txt).apply(self.lang.QuestionMarkInQuotationRule,
+                              *self.lang.ExclamationPointRules.All)
+        txt = ListItemReplacer(txt).replace_parens()
+        txt = self.sentence_boundary_punctuation(txt)
+        return txt
+
+    def replace_numbers(self):
+        self.text = Text(self.text).apply(*self.lang.Numbers.All)
+
+    def abbreviations_replacer(self):
+        if hasattr(self.lang, "AbbreviationReplacer"):
+            return self.lang.AbbreviationReplacer(self.text, self.lang)
+        else:
+            return AbbreviationReplacer(self.text, self.lang)
+
+    def replace_abbreviations(self):
+        self.text = self.abbreviations_replacer().replace()
+
+    def between_punctuation_processor(self, txt):
+        if hasattr(self.lang, "BetweenPunctuation"):
+            return self.lang.BetweenPunctuation(txt)
+        else:
+            return BetweenPunctuation(txt)
+
+    def between_punctuation(self, txt):
+        txt = self.between_punctuation_processor(txt).replace()
+        return txt
+
+    def sentence_boundary_punctuation(self, txt):
+        if hasattr(self.lang, 'ReplaceColonBetweenNumbersRule'):
+            txt = Text(txt).apply(
+                self.lang.ReplaceColonBetweenNumbersRule)
+        if hasattr(self.lang, 'ReplaceNonSentenceBoundaryCommaRule'):
+            txt = Text(txt).apply(
+                self.lang.ReplaceNonSentenceBoundaryCommaRule)
+        # retain exclamation mark if it is an ending character of a given text
+        txt = re.sub(r'&ᓴ&$', '!', txt)
+        txt = [
+            m.group() for m in re.finditer(self.lang.SENTENCE_BOUNDARY_REGEX, txt)
+            ]
+        return txt

pysbd/punctuation_replacer.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+import re
+from pysbd.utils import Rule, Text
+
+
+class EscapeRegexReservedCharacters(object):
+    LeftParen = Rule(r'\(', '\\(')
+    RightParen = Rule(r'\)', '\\)')
+    # LeftParen = Rule(re.escape(r'('), '(')
+    # RightParen = Rule(re.escape(r')'), ')')
+    LeftBracket = Rule(r'\[', '\\[')
+    RightBracket = Rule(r'\]', '\\]')
+    Dash = Rule(r'\-', '\\-')
+
+    All = [LeftParen, RightParen, LeftBracket, RightBracket, Dash]
+
+
+class SubEscapedRegexReservedCharacters(object):
+    SubLeftParen = Rule(r'\\\(', '(')
+    SubRightParen = Rule(r'\\\)', ')')
+    # SubLeftParen = Rule(re.escape(r"\\("), "(")
+    # SubRightParen = Rule(re.escape(r'\\)'), ')')
+    SubLeftBracket = Rule(r'\\\[', '[')
+    SubRightBracket = Rule(r'\\\]', ']')
+    SubDash = Rule(r'\\\-', '-')
+
+    All = [
+        SubLeftParen, SubRightParen, SubLeftBracket, SubRightBracket, SubDash
+    ]
+
+
+def replace_punctuation(match, match_type=None):
+    text = Text(match.group()).apply(*EscapeRegexReservedCharacters.All)
+    sub = re.sub(r'\.', '∯', text)
+    sub = re.sub(r'։', '⍟', sub) # ADDED FOR ARMENIAN
+    sub_1 = re.sub(r'\。', '&ᓰ&', sub)
+    sub_2 = re.sub(r'\．', '&ᓱ&', sub_1)
+    sub_3 = re.sub(r'\！', '&ᓳ&', sub_2)
+    sub_4 = re.sub(r'\!', '&ᓴ&', sub_3)
+    sub_5 = re.sub(r'\?', '&ᓷ&', sub_4)
+    last_sub = re.sub(r'\？', '&ᓸ&', sub_5)
+    if match_type != 'single':
+        last_sub = re.sub(r"'", '&⎋&', last_sub)
+    text = Text(last_sub).apply(*SubEscapedRegexReservedCharacters.All)
+    return text

pysbd/segmenter.py

@@ -0,0 +1,96 @@
+# -*- coding: utf-8 -*-
+import re
+
+from pysbd.languages import Language
+from pysbd.processor import Processor
+from pysbd.cleaner import Cleaner
+from pysbd.utils import TextSpan
+
+class Segmenter(object):
+
+    def __init__(self, language="en", clean=False, doc_type=None, char_span=False):
+        """Segments a text into an list of sentences
+        with or withour character offsets from original text
+
+        Parameters
+        ----------
+        language : str, required
+            specify a language use its two character ISO 639-1 code,
+            by default "en"
+        clean : bool, optional
+            cleans original text, by default False
+        doc_type : [type], optional
+            Normal text or OCRed text, by default None
+            set to `pdf` for OCRed text
+        char_span : bool, optional
+            Get start & end character offsets of each sentences
+            within original text, by default False
+        """
+        self.language = language
+        self.language_module = Language.get_language_code(language)
+        self.clean = clean
+        self.doc_type = doc_type
+        self.char_span = char_span
+        if self.clean and self.char_span:
+            raise ValueError("char_span must be False if clean is True. "
+                             "Since `clean=True` will modify original text.")
+        # when doctype is pdf then force user to clean the text
+        # char_span func wont be provided with pdf doctype also
+        elif self.doc_type == 'pdf' and not self.clean:
+            raise ValueError("`doc_type='pdf'` should have `clean=True` & "
+                            "`char_span` should be False since original"
+                            "text will be modified.")
+
+    def cleaner(self, text):
+        if hasattr(self.language_module, "Cleaner"):
+            return self.language_module.Cleaner(text, self.language_module,
+                                                doc_type=self.doc_type)
+        else:
+            return Cleaner(text, self.language_module, doc_type=self.doc_type)
+
+    def processor(self, text):
+        if hasattr(self.language_module, "Processor"):
+            return self.language_module.Processor(text, self.language_module,
+                                                  char_span=self.char_span)
+        else:
+            return Processor(text, self.language_module,
+                             char_span=self.char_span)
+
+    def sentences_with_char_spans(self, sentences):
+        # since SENTENCE_BOUNDARY_REGEX doesnt account
+        # for trailing whitespaces \s* & is used as suffix
+        # to keep non-destructive text after segments joins
+        sent_spans = []
+        prior_end_char_idx = 0
+        for sent in sentences:
+            for match in re.finditer('{0}\s*'.format(re.escape(sent)), self.original_text):
+                match_str = match.group()
+                match_start_idx, match_end_idx = match.span()
+                if match_end_idx > prior_end_char_idx:
+                    # making sure if curren sentence and its span
+                    # is either first sentence along with its char spans
+                    # or current sent spans adjacent to prior sentence spans
+                    sent_spans.append(
+                        TextSpan(match_str, match_start_idx, match_end_idx))
+                    prior_end_char_idx = match_end_idx
+                    break
+        return sent_spans
+
+    def segment(self, text):
+        self.original_text = text
+        if not text:
+            return []
+
+        if self.clean or self.doc_type == 'pdf':
+            text = self.cleaner(text).clean()
+
+        postprocessed_sents = self.processor(text).process()
+        sentence_w_char_spans = self.sentences_with_char_spans(postprocessed_sents)
+        if self.char_span:
+            return sentence_w_char_spans
+        elif self.clean:
+            # clean and destructed sentences
+            return postprocessed_sents
+        else:
+            # nondestructive with whitespaces
+            return [textspan.sent for textspan in sentence_w_char_spans]

pysbd/utils.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import re
+import pysbd
+
+class Rule(object):
+
+    def __init__(self, pattern, replacement):
+        self.pattern = pattern
+        self.replacement = replacement
+
+    def __repr__(self):  # pragma: no cover
+        return '<{} pattern="{}" and replacement="{}">'.format(
+            self.__class__.__name__, self.pattern, self.replacement)
+
+
+class Text(str):
+    """Extending str functionality to apply regex rules
+
+    https://stackoverflow.com/questions/4698493/can-i-add-custom-methods-attributes-to-built-in-python-types
+
+    Parameters
+    ----------
+    str : str
+        string content
+
+    Returns
+    -------
+    str
+        input as it is if rule pattern doesnt match
+        else replacing found pattern with replacement chars
+    """
+    def apply(self, *rules):
+        for each_r in rules:
+            self = re.sub(each_r.pattern, each_r.replacement, self)
+        return self
+
+
+class TextSpan(object):
+
+    def __init__(self, sent, start, end):
+        """
+        Sentence text and its start & end character offsets within original text
+
+        Parameters
+        ----------
+        sent : str
+            Sentence text
+        start : int
+            start character offset of a sentence in original text
+        end : int
+            end character offset of a sentence in original text
+        """
+        self.sent = sent
+        self.start = start
+        self.end = end
+
+    def __repr__(self):  # pragma: no cover
+        return "{0}(sent={1}, start={2}, end={3})".format(
+            self.__class__.__name__, repr(self.sent), self.start, self.end)
+
+    def __eq__(self, other):
+        if isinstance(self, other.__class__):
+            return self.sent == other.sent and self.start == other.start and self.end == other.end
+
+
+class PySBDFactory(object):
+    """pysbd as a spacy component through entrypoints"""
+
+    def __init__(self, nlp, language='en'):
+        self.nlp = nlp
+        self.seg = pysbd.Segmenter(language=language, clean=False,
+                                   char_span=True)
+
+    def __call__(self, doc):
+        sents_char_spans = self.seg.segment(doc.text_with_ws)
+        start_token_ids = [sent.start for sent in sents_char_spans]
+        for token in doc:
+            token.is_sent_start = (True if token.idx
+                                   in start_token_ids else False)
+        return doc