Initial commit

.github/workflows/cicd.yaml

@@ -0,0 +1,17 @@
+# .github/workflows/sync-to-hub.yml
+name: Sync to Hugging Face Hub
+on:
+  push:
+    branches: [main]
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Push to hub
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: |
+          git push https://YOUR_USERNAME:[email protected]/spaces/YOUR_USERNAME/nigerian-language-generator-space main
+

.gitignore

@@ -0,0 +1,10 @@
+__pycache__/
+*.pyc
+.env
+venv/
+*.pth
+.DS_Store
+outputs/
+logs/
+data/raw/
+data/processed/

Dockerfile

@@ -0,0 +1,10 @@
+FROM python:3.9-slim
+
+WORKDIR /code
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+CMD ["python", "app.py"]

README.md

@@ -1,14 +0,0 @@
----
-title: Nigerian Languages
-emoji: 📉
-colorFrom: purple
-colorTo: yellow
-sdk: gradio
-sdk_version: 5.11.0
-app_file: app.py
-pinned: false
-license: apache-2.0
-short_description: Nigerian_languages
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app/interface/gardio_app.py

@@ -0,0 +1,28 @@
+import gradio as gr
+from app.model.model import NigerianLanguageModel
+from app.model.config import ModelConfig
+
+# Initialize model
+config = ModelConfig()
+model = NigerianLanguageModel(config)
+
+# Create interface
+def generate_text(prompt: str, language: str) -> str:
+    tagged_prompt = f"[{language.upper()}] {prompt}"
+    return model.generate(tagged_prompt)
+
+# Define Gradio interface
+interface = gr.Interface(
+    fn=generate_text,
+    inputs=[
+        gr.Textbox(label="Enter your prompt"),
+        gr.Dropdown(choices=["YORUBA", "IGBO", "HAUSA"], label="Select Language")
+    ],
+    outputs=gr.Textbox(label="Generated Text"),
+    title="Nigerian Language Generator",
+    description="Generate text in Yoruba, Igbo, or Hausa using a fine-tuned GPT model."
+)
+
+# Start the interface
+if __name__ == "__main__":
+    interface.launch()

app/model/config.py

@@ -0,0 +1,14 @@
+from dataclasses import dataclass
+from typing import List, Optional
+import torch
+
+@dataclass
+class ModelConfig:
+    model_name: str = "gpt2"
+    max_length: int = 128
+    batch_size: int = 16
+    learning_rate: float = 2e-5
+    num_train_epochs: int = 3
+    languages: List[str] = ("YORUBA", "IGBO", "HAUSA")
+    device: str = "cuda" if torch.cuda.is_available() else "cpu"
+    output_dir: str = "outputs"

app/model/model.py

@@ -0,0 +1,38 @@
+# 1. app/model/config.py
+from dataclasses import dataclass
+from typing import List, Optional
+import torch
+
+@dataclass
+class ModelConfig:
+    model_name: str = "gpt2"
+    max_length: int = 128
+    batch_size: int = 16
+    learning_rate: float = 2e-5
+    num_train_epochs: int = 3
+    languages: List[str] = ("YORUBA", "IGBO", "HAUSA")
+    device: str = "cuda" if torch.cuda.is_available() else "cpu"
+    output_dir: str = "outputs"
+
+# app/model/model.py
+from transformers import AutoTokenizer, AutoModelForCausalLM
+import torch
+from .config import ModelConfig
+
+class NigerianLanguageModel:
+    def __init__(self, config: ModelConfig):
+        self.config = config
+        self.setup_model()
+    
+    def setup_model(self):
+        self.tokenizer = AutoTokenizer.from_pretrained(self.config.model_name)
+        self.model = AutoModelForCausalLM.from_pretrained(self.config.model_name)
+        self._setup_special_tokens()
+        self.model.to(self.config.device)
+
+    def _setup_special_tokens(self):
+        special_tokens = {
+            "additional_special_tokens": [f"[{lang}]" for lang in self.config.languages]
+        }
+        self.tokenizer.add_special_tokens(special_tokens)
+        self.model.resize_token_embeddings(len(self.tokenizer))

app/model/tokenizer.py

@@ -0,0 +1,14 @@
+from transformers import PreTrainedTokenizerFast
+from typing import List, Dict
+
+class NigerianLanguageTokenizer:
+    def __init__(self, base_tokenizer: PreTrainedTokenizerFast):
+        self.tokenizer = base_tokenizer
+        
+    def tokenize_batch(self, texts: List[str]) -> Dict:
+        return self.tokenizer(
+            texts,
+            padding=True,
+            truncation=True,
+            return_tensors="pt"
+        )

app/utils/data_processing.py

@@ -0,0 +1,13 @@
+import pandas as pd
+from typing import List, Dict
+import os
+
+def load_language_data(data_dir: str, language: str) -> List[str]:
+    filepath = os.path.join(data_dir, f"{language.lower()}/texts.txt")
+    with open(filepath, 'r', encoding='utf-8') as f:
+        return f.readlines()
+
+def preprocess_text(text: str) -> str:
+    text = text.strip()
+    text = ' '.join(text.split())
+    return text

app/utils/text_processing.py

@@ -0,0 +1,10 @@
+import re
+from typing import List
+
+def clean_text(text: str) -> str:
+    text = re.sub(r'\s+', ' ', text)
+    text = text.strip()
+    return text
+
+def split_into_sentences(text: str) -> List[str]:
+    return [s.strip() for s in re.split(r'[.!?]+', text) if s.strip()]

configs/model_config.yaml

@@ -0,0 +1,10 @@
+model:
+  name: "gpt2"
+  max_length: 128
+  batch_size: 16
+  learning_rate: 2e-5
+  num_train_epochs: 3
+  languages:
+    - YORUBA
+    - IGBO
+    - HAUSA

configs/training_config.yaml

@@ -0,0 +1,10 @@
+training:
+  output_dir: "outputs"
+  evaluation_strategy: "steps"
+  eval_steps: 500
+  save_steps: 500
+  logging_steps: 100
+  learning_rate: 2e-5
+  num_train_epochs: 3
+  per_device_train_batch_size: 16
+  per_device_eval_batch_size: 16

requirements.txt

@@ -0,0 +1,7 @@
+transformers>=4.30.0
+torch>=2.0.0
+gradio>=3.50.0
+datasets>=2.14.0
+pandas>=1.5.0
+pytest>=7.0.0
+pyyaml>=6.0.0

scripts/evaluate.py

@@ -0,0 +1,13 @@
+from app.model.model import NigerianLanguageModel
+import torch
+from typing import Dict
+import json
+
+def evaluate_model(model: NigerianLanguageModel, test_data) -> Dict:
+    results = {
+        "perplexity": [],
+        "generation_samples": []
+    }
+    
+    # Add evaluation logic here
+    return results

scripts/train.py

@@ -0,0 +1,38 @@
+import torch
+from transformers import Trainer, TrainingArguments
+from app.model.model import NigerianLanguageModel
+from app.model.config import ModelConfig
+
+def train_model(model: NigerianLanguageModel, train_dataset, eval_dataset=None):
+    training_args = TrainingArguments(
+        output_dir="outputs",
+        num_train_epochs=model.config.num_train_epochs,
+        per_device_train_batch_size=model.config.batch_size,
+        learning_rate=model.config.learning_rate,
+        save_steps=500,
+    )
+    
+    trainer = Trainer(
+        model=model.model,
+        args=training_args,
+        train_dataset=train_dataset,
+        eval_dataset=eval_dataset
+    )
+    
+    trainer.train()
+
+# scripts/preprocess.py
+from app.utils.data_preprocessing import load_language_data, preprocess_text
+import os
+
+def main():
+    languages = ["yoruba", "igbo", "hausa"]
+    for lang in languages:
+        data = load_language_data("data/raw", lang)
+        processed_data = [preprocess_text(text) for text in data]
+        
+        output_dir = f"data/processed/{lang}"
+        os.makedirs(output_dir, exist_ok=True)
+        
+        with open(f"{output_dir}/processed_texts.txt", 'w', encoding='utf-8') as f:
+            f.writelines(processed_data)

setup.py

@@ -0,0 +1,13 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="nigerian-language-generator",
+    version="0.1.0",
+    packages=find_packages(),
+    install_requires=[
+        "transformers>=4.30.0",
+        "torch>=2.0.0",
+        "gradio>=3.50.0",
+        "datasets>=2.14.0",
+    ],
+)

tests/test_model.py

@@ -0,0 +1,9 @@
+import pytest
+from app.model.model import NigerianLanguageModel
+from app.model.config import ModelConfig
+
+def test_model_initialization():
+    config = ModelConfig()
+    model = NigerianLanguageModel(config)
+    assert model.tokenizer is not None
+    assert model.model is not None

tests/test_processing.py

@@ -0,0 +1,7 @@
+import pytest
+from app.utils.data_preprocessing import preprocess_text
+
+def test_preprocess_text():
+    text = "  Sample   text  with   spaces  "
+    processed = preprocess_text(text)
+    assert processed == "Sample text with spaces"

tests/test_tokenizer.py

@@ -0,0 +1,10 @@
+import pytest
+from app.model.tokenizer import NigerianLanguageTokenizer
+from transformers import AutoTokenizer
+
+def test_tokenizer():
+    base_tokenizer = AutoTokenizer.from_pretrained("gpt2")
+    tokenizer = NigerianLanguageTokenizer(base_tokenizer)
+    text = "Sample text"
+    tokens = tokenizer.tokenize_batch([text])
+    assert tokens is not None