Add initial files

Dockerfile

@@ -0,0 +1,28 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements first to leverage Docker cache
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application code
+COPY . .
+
+# Create uploads directory
+RUN mkdir -p uploads && chmod 777 uploads
+
+# Create a non-root user
+RUN useradd -m appuser && chown -R appuser:appuser /app
+USER appuser
+
+# # Set environment variables for the buffered output
+# ENV PYTHONUNBUFFERED=1
+
+# Default command (can be overridden in docker-compose.yml)
+CMD ["gunicorn", "--config", "gunicorn_config.py", "app:app"]

Dockerfile.hf

@@ -0,0 +1,61 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements first to leverage Docker cache
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application code
+COPY . .
+
+# Create uploads directory
+RUN mkdir -p uploads && chmod 777 uploads
+
+# Create a non-root user
+RUN useradd -m appuser && chown -R appuser:appuser /app
+USER appuser
+
+# Set environment variables
+ENV FLASK_ENV=production
+ENV REDIS_URL=redis://localhost:6379/0
+
+# Expose ports
+EXPOSE 7860
+EXPOSE 6379
+EXPOSE 5555
+
+# Install and setup Redis
+USER root
+RUN apt-get update && apt-get install -y redis-server \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create start script
+COPY <<EOF /app/start.sh
+#!/bin/bash
+# Start Redis
+redis-server --daemonize yes
+
+# Start Celery worker
+celery -A app.celery worker --loglevel=info &
+
+# Start Flower
+celery -A app.celery flower --port=5555 &
+
+# Start the Flask application with gunicorn
+exec gunicorn --config gunicorn_config.py --bind 0.0.0.0:7860 app:app
+EOF
+
+RUN chmod +x /app/start.sh
+
+# Switch back to non-root user
+USER appuser
+
+# Start all services
+CMD ["/app/start.sh"]

README.md

@@ -1,11 +1,53 @@
----
-title: Selector
-emoji: 🦀
-colorFrom: gray
-colorTo: gray
-sdk: docker
-pinned: false
-license: gpl-3.0
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Selector Server
+
+A web interface for the Selector library, providing tools for subset selection and diversity calculations.
+
+## Local Development
+
+Run the server locally using Docker Compose:
+
+```bash
+docker compose up --build --scale celery_worker=3 -d
+```
+
+The server will be available at:
+- Main application: http://localhost:8008
+- Flower monitoring: http://localhost:5555
+
+## Hugging Face Deployment
+
+1. Build the Hugging Face specific Docker image:
+```bash
+docker build -t selector-server-hf -f Dockerfile.hf .
+```
+
+2. Login to Hugging Face CLI:
+```bash
+huggingface-cli login
+```
+
+3. Create a new Space on Hugging Face:
+- Go to https://huggingface.co/spaces
+- Click "Create new Space"
+- Choose "Docker" as the Space SDK
+- Set the hardware requirements (recommended: CPU + 16GB RAM)
+
+4. Push the Docker image:
+```bash
+# Tag the image
+docker tag selector-server-hf registry.hf.space/your-username/your-space-name:latest
+
+# Push to Hugging Face
+docker push registry.hf.space/your-username/your-space-name:latest
+```
+
+The application will be available at: https://huggingface.co/spaces/your-username/your-space-name
+
+## Features
+
+- Upload feature matrices for subset selection
+- Calculate diversity metrics
+- Visualize results
+- RESTful API for programmatic access
+- Celery-based task queue for long-running calculations
+- Real-time task monitoring with Flower

app.py

@@ -0,0 +1,555 @@
+# import json
+import inspect
+import io
+import os
+import shutil
+import tempfile
+import threading
+import uuid
+import warnings
+from datetime import datetime
+from typing import Callable, Dict
+
+import markdown
+import matplotlib.pyplot as plt
+import numpy as np
+import orjson
+import pandas as pd
+from flask import Flask, Response, render_template, request, send_file
+from flask_status import FlaskStatus
+from selector.methods.distance import DISE, MaxMin, MaxSum, OptiSim
+from selector.methods.partition import GridPartition, Medoid
+from selector.methods.similarity import NSimilarity
+from selector.measures.diversity import compute_diversity
+from sklearn.metrics import pairwise_distances
+from werkzeug.utils import secure_filename
+
+try:
+    from celery_config import celery
+
+    CELERY_AVAILABLE = True
+except ImportError:
+    CELERY_AVAILABLE = False
+
+app = Flask(__name__)
+app_status = FlaskStatus(app)
+app.config["MAX_CONTENT_LENGTH"] = 32 * 1024 * 1024  # 32MB max file size
+app.config["UPLOAD_FOLDER"] = "uploads"
+file_lock = threading.Lock()
+
+# Ensure upload directory exists
+os.makedirs(app.config["UPLOAD_FOLDER"], exist_ok=True)
+
+ALLOWED_EXTENSIONS = {"txt", "npz", "xlsx", "xls"}
+
+# Map algorithm names to their functions
+SELECTION_ALGORITHM_MAP = {
+    # Distance-based methods
+    "MaxMin": MaxMin,
+    "MaxSum": MaxSum,
+    "OptiSim": OptiSim,
+    "DISE": DISE,
+    # Partition-based methods
+    "GridPartition": GridPartition,
+    # Similarity-based methods
+    "NSimilarity": NSimilarity,
+}
+
+
+def allowed_file(filename):
+    return "." in filename and filename.rsplit(".", 1)[1].lower() in ALLOWED_EXTENSIONS
+
+
+def get_unique_upload_dir():
+    """Create a unique directory for each upload session."""
+    unique_dir = os.path.join(app.config["UPLOAD_FOLDER"], str(uuid.uuid4()))
+    os.makedirs(unique_dir, exist_ok=True)
+    return unique_dir
+
+
+def clean_upload_dir(directory):
+    """Safely clean up upload directory."""
+    try:
+        if os.path.exists(directory):
+            shutil.rmtree(directory)
+    except Exception as e:
+        print(f"Error cleaning directory {directory}: {e}")
+
+
+def load_data(filepath):
+    """Load data from various file formats."""
+    try:
+        ext = filepath.rsplit(".", 1)[1].lower()
+        if ext == "npz":
+            with np.load(filepath) as data:
+                return data["arr_0"] if "arr_0" in data else next(iter(data.values()))
+        elif ext == "txt":
+            return np.loadtxt(filepath)
+        elif ext in ["xlsx", "xls"]:
+            df = pd.read_excel(filepath)
+            return df.to_numpy()
+    except Exception as e:
+        raise ValueError(f"Error loading file {filepath}: {str(e)}")
+
+
+def create_json_response(data, status=200):
+    """Create a JSON response using orjson for better numpy array handling"""
+    return Response(
+        orjson.dumps(data, option=orjson.OPT_SERIALIZE_NUMPY, default=str),
+        status=status,
+        mimetype="application/json",
+    )
+
+
+def read_markdown_file(filename):
+    """Read and convert markdown file to HTML."""
+    filepath = os.path.join(os.path.dirname(__file__), "md_files", filename)
+    try:
+        with open(filepath, "r", encoding="utf-8") as f:
+            content = f.read()
+
+            # Pre-process math blocks to protect them
+            # content = content.replace('\\\\', '\\\\\\\\')  # Escape backslashes in math
+
+            # Convert markdown to HTML with math and table support
+            md = markdown.Markdown(extensions=["tables", "fenced_code", "codehilite", "attr_list"])
+
+            # First pass: convert markdown to HTML
+            html = md.convert(content)
+
+            # Post-process math blocks
+            # Handle display math ($$...$$)
+            html = html.replace("<p>$$", '<div class="math-block">$$')
+            html = html.replace("$$</p>", "$$</div>")
+
+            # Handle inline math ($...$)
+            # We don't need special handling for inline math as MathJax will handle it
+
+            return html
+    except Exception as e:
+        print(f"Error reading markdown file {filename}: {e}")
+        return f"<p>Error loading content: {str(e)}</p>"
+
+
+def get_default_parameters(func):
+    """Get default parameters for a function from its signature."""
+    sig = inspect.signature(func)
+    defaults = {}
+
+    for name, param in sig.parameters.items():
+        if name == "self" or name == "fun_dist":  # Skip self and dist_metric
+            continue
+        if param.default is not param.empty:
+            defaults[name] = param.default
+
+    return defaults
+
+
+@app.route("/get_default_params/<algorithm>")
+def get_default_params(algorithm):
+    """API endpoint to get default parameters for an algorithm."""
+    if algorithm not in SELECTION_ALGORITHM_MAP:
+        return create_json_response({"error": f"Unknown algorithm: {algorithm}"}, 400)
+
+    try:
+        # Get the algorithm class
+        algorithm_class = SELECTION_ALGORITHM_MAP[algorithm]
+        # Get default parameters from __init__
+        params = get_default_parameters(algorithm_class.__init__)
+        return create_json_response(params)
+    except Exception as e:
+        return create_json_response({"error": f"Error getting parameters: {str(e)}"}, 500)
+
+
+@app.route("/get_default_selection_params/<algorithm>")
+def get_default_selection_params(algorithm):
+    """API endpoint to get default parameters for a selection algorithm."""
+    if algorithm not in SELECTION_ALGORITHM_MAP:
+        return create_json_response({"error": f"Algorithm unsupported: {algorithm}"}, 400)
+
+    try:
+        return create_json_response(get_default_selection_params(algorithm))
+    except Exception as e:
+        return create_json_response({"error": f"Error getting parameters: {str(e)}"}, 500)
+
+
+@app.route("/")
+def home():
+    return render_template("index.html")
+
+
+@app.route("/md/<filename>")
+def get_markdown(filename):
+    """Serve markdown files as HTML."""
+    if not filename.endswith(".md"):
+        filename = filename + ".md"
+    html = read_markdown_file(filename)
+    return create_json_response({"html": html})
+
+
+def process_selection(arr, algorithm, parameters, dist_metric):
+    """
+    Process feature matrix using the specified selection algorithm.
+
+    Parameters
+    ----------
+    arr : np.ndarray
+        Input feature matrix
+    algorithm : str
+        Name of the selection algorithm to use
+    parameters : dict
+        Parameters for the algorithm
+    dist_metric : str, optional
+        Distance function to use.
+
+    Returns
+    -------
+    dict
+        Dictionary containing results and any warnings
+    """
+    result = {"success": False, "error": None, "warnings": [], "indices": None}
+
+    try:
+        # Get the algorithm class
+        algorithm_class = SELECTION_ALGORITHM_MAP.get(algorithm)
+
+        if algorithm_class is None:
+            raise ValueError(f"Unknown algorithm: {algorithm}")
+
+        # Get size parameter
+        size = parameters.pop('size', None)
+        if size is None:
+            raise ValueError("Subset size must be specified")
+
+        try:
+            size = int(size)
+            if size < 1:
+                raise ValueError
+        except (TypeError, ValueError):
+            raise ValueError("Subset size must be a positive integer")
+
+        # Validate size against array dimensions
+        if size > arr.shape[0]:
+            raise ValueError(f"Subset size ({size}) cannot be larger than the number of samples ({arr.shape[0]})")
+
+        # Handle distance-based methods differently
+        is_distance_based = algorithm in ["MaxMin", "MaxSum", "OptiSim", "DISE"]
+
+        # Convert array to float for computations
+        arr_float = arr.astype(float)
+
+        # Compute or prepare the input matrix
+        if is_distance_based:
+            # For distance-based methods, compute distance matrix
+            try:
+                if dist_metric and dist_metric != "":
+                    # Use specified distance metric
+                    arr_dist = pairwise_distances(arr_float, metric=dist_metric)
+                else:
+                    # Default to euclidean distance
+                    arr_dist = pairwise_distances(arr_float, metric='euclidean')
+            except Exception as e:
+                raise ValueError(f"Error computing distance matrix: {str(e)}")
+        else:
+            # For non-distance-based methods, use the original float array
+            arr_dist = arr_float
+
+        # Handle special case for GridPartition
+        if algorithm == "GridPartition":
+            # Ensure nbins_axis is provided and is an integer
+            nbins_axis = parameters.get('nbins_axis')
+            if nbins_axis is None:
+                raise ValueError("nbins_axis must be specified for GridPartition")
+            try:
+                parameters['nbins_axis'] = int(nbins_axis)
+                if parameters['nbins_axis'] < 1:
+                    raise ValueError
+            except (TypeError, ValueError):
+                raise ValueError("nbins_axis must be a positive integer")
+
+        # Initialize and run the algorithm
+        try:
+            collector = algorithm_class(**parameters)
+            indices = collector.select(arr_dist, size=size)
+
+            # Ensure indices are valid
+            if indices is None:
+                raise ValueError("Algorithm returned None instead of indices")
+            if len(indices) != size:
+                warnings.warn(f"Algorithm returned {len(indices)} indices but expected {size}")
+
+            # Convert indices to list and validate
+            indices_list = indices.tolist() if isinstance(indices, np.ndarray) else list(indices)
+            if not all(isinstance(i, (int, np.integer)) and 0 <= i < arr.shape[0] for i in indices_list):
+                raise ValueError("Algorithm returned invalid indices")
+
+            result["success"] = True
+            result["indices"] = indices_list
+
+        except Exception as e:
+            import traceback
+            print(f"Traceback: {traceback.format_exc()}")
+            raise ValueError(f"Error executing algorithm: {str(e)}")
+
+    except Warning as w:
+        result["warnings"].append(str(w))
+    except Exception as e:
+        result["error"] = str(e)
+
+    return result
+
+
+@app.route("/upload_selection", methods=["POST"])
+def upload_selection_file():
+    """Handle file upload and process selection."""
+    try:
+        print("Debug - Starting upload_selection_file")
+
+        if "file" not in request.files:
+            return create_json_response({"error": "No file provided"}, 400)
+
+        file = request.files["file"]
+        if file.filename == "":
+            return create_json_response({"error": "No file selected"}, 400)
+
+        if not allowed_file(file.filename):
+            return create_json_response({"error": "File type not allowed"}, 400)
+
+        # Get parameters
+        algorithm = request.form.get("algorithm")
+        if not algorithm:
+            return create_json_response({"error": "No algorithm specified"}, 400)
+
+        # Get size parameter
+        size = request.form.get("size")
+        if not size:
+            return create_json_response({"error": "Subset size must be specified"}, 400)
+
+        # Get distance function
+        dist_metric = request.form.get("func_dist", "")
+
+        # Parse parameters
+        try:
+            parameters = orjson.loads(request.form.get("parameters", "{}"))
+        except Exception as e:
+            parameters = {}
+
+        # Add size to parameters
+        parameters["size"] = size
+
+        # Create a unique directory for this upload
+        upload_dir = get_unique_upload_dir()
+
+        try:
+            # Save file with unique name
+            file_path = os.path.join(
+                upload_dir, secure_filename(str(uuid.uuid4()) + "_" + file.filename)
+            )
+
+            with file_lock:
+                file.save(file_path)
+
+            # Load data
+            array = load_data(file_path)
+
+            # Process the selection with separate dist_metric parameter
+            result = process_selection(array, algorithm, parameters, dist_metric)
+
+            return create_json_response(result)
+
+        except Exception as e:
+            return create_json_response({"error": str(e)}, 500)
+
+        finally:
+            # Clean up the unique upload directory
+            clean_upload_dir(upload_dir)
+
+    except Exception as e:
+        return create_json_response({"error": f"Error processing request: {str(e)}"}, 400)
+
+
+@app.route("/download", methods=["POST"])
+def download():
+    """Download selected indices in specified format."""
+    try:
+        data = request.get_json()
+        if not data or "indices" not in data:
+            return create_json_response({"error": "No indices provided"}, 400)
+
+        indices = data["indices"]
+        format = data.get("format", "txt")
+        timestamp = data.get("timestamp", datetime.now().strftime("%Y%m%d-%H%M%S"))
+
+        # Create a BytesIO buffer for the file
+        buffer = io.BytesIO()
+
+        # Define format-specific settings
+        format_settings = {
+            "txt": {
+                "extension": "txt",
+                "mimetype": "text/plain",
+                "processor": lambda b, d: b.write("\n".join(map(str, d)).encode()),
+            },
+            "npz": {
+                "extension": "npz",
+                "mimetype": "application/octet-stream",
+                "processor": lambda b, d: np.savez_compressed(b, indices=np.array(d)),
+            },
+            "xlsx": {
+                "extension": "xlsx",
+                "mimetype": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+                "processor": lambda b, d: pd.DataFrame({"selected_indices": d}).to_excel(
+                    b, index=False
+                ),
+            },
+        }
+
+        if format not in format_settings:
+            return create_json_response({"error": f"Unsupported format: {format}"}, 400)
+
+        settings = format_settings[format]
+
+        # Process the file
+        settings["processor"](buffer, indices)
+
+        # Create filename with correct extension
+        filename = f'selected_indices_{timestamp}.{settings["extension"]}'
+
+        # Seek to beginning of file
+        buffer.seek(0)
+
+        return send_file(
+            buffer, mimetype=settings["mimetype"], as_attachment=True, download_name=filename
+        )
+
+    except Exception as e:
+        print(f"Error in download: {str(e)}")
+        return create_json_response({"error": str(e)}, 500)
+
+
+@app.route("/calculate_diversity", methods=["POST"])
+def calculate_diversity():
+    """Calculate diversity score for the given feature subset."""
+    try:
+        # Get files from request
+        feature_subset_file = request.files.get('feature_subset')
+        features_file = request.files.get('features')
+        
+        if not feature_subset_file:
+            return create_json_response({"error": "Feature subset file is required"}, 400)
+
+        # Get other parameters
+        div_type = request.form.get('div_type', 'shannon_entropy')
+        div_parameters = orjson.loads(request.form.get('div_parameters', '{}'))
+
+        # Read feature subset
+        try:
+            # Save the uploaded file
+            feature_subset_path = os.path.join(app.config['UPLOAD_FOLDER'], secure_filename(feature_subset_file.filename))
+            feature_subset_file.save(feature_subset_path)
+
+            # Read the feature subset file
+            feature_subset = load_data(feature_subset_path)
+            if feature_subset is None:
+                raise ValueError(f"Failed to read feature subset file: {feature_subset_file.filename}")
+
+            # Convert to float array
+            feature_subset = feature_subset.astype(float)
+
+            # Clean up the temporary file
+            os.remove(feature_subset_path)
+        except Exception as e:
+            return create_json_response({"error": f"Error reading feature subset file: {str(e)}"}, 400)
+
+        # Read features if provided
+        features = None
+        if features_file:
+            try:
+                # Save the uploaded file
+                features_path = os.path.join(app.config['UPLOAD_FOLDER'], secure_filename(features_file.filename))
+                features_file.save(features_path)
+
+                # Read the features file
+                features = load_data(features_path)
+                if features is None:
+                    raise ValueError(f"Failed to read features file: {features_file.filename}")
+
+                # Convert to float array
+                features = features.astype(float)
+
+                # Clean up the temporary file
+                os.remove(features_path)
+            except Exception as e:
+                return create_json_response({"error": f"Error reading features file: {str(e)}"}, 400)
+
+        # Extract parameters
+        normalize = div_parameters.get('normalize', False)
+        truncation = div_parameters.get('truncation', False)
+        cs = div_parameters.get('cs', None)
+
+        # Calculate diversity
+        try:
+            diversity_score = compute_diversity(
+                feature_subset=feature_subset,
+                div_type=div_type,
+                normalize=normalize,
+                truncation=truncation,
+                features=features,
+                cs=cs
+            )
+            
+            return create_json_response({
+                "success": True,
+                "diversity_score": float(diversity_score)
+            })
+
+        except Exception as e:
+            import traceback
+            print(f"Error calculating diversity: {str(e)}")
+            print(f"Traceback: {traceback.format_exc()}")
+            return create_json_response({"error": f"Error calculating diversity: {str(e)}"}, 400)
+
+    except Exception as e:
+        return create_json_response({"error": str(e)}, 500)
+
+
+@app.route("/status")
+def server_status():
+    """Return server status"""
+    status = {
+        "status": "ok",
+        "message": "Server is running",
+        "timestamp": datetime.now().isoformat(),
+        "components": {"flask": True, "celery": False, "redis": False},
+    }
+
+    if CELERY_AVAILABLE:
+        # Check Celery
+        try:
+            celery.control.ping(timeout=1)
+            status["components"]["celery"] = True
+        except Exception as e:
+            print(f"Celery check failed: {e}")
+
+        # Check Redis
+        try:
+            redis_client = celery.backend.client
+            redis_client.ping()
+            status["components"]["redis"] = True
+        except Exception as e:
+            print(f"Redis check failed: {e}")
+
+        # Set overall status
+        if not all(status["components"].values()):
+            status["status"] = "degraded"
+            status["message"] = "Some components are not available"
+    else:
+        status["message"] = "Running without Celery/Redis support"
+
+    return create_json_response(status)
+
+
+if __name__ == "__main__":
+    app.run(debug=True, host="0.0.0.0", port=8008)
+    from flask_debugtoolbar import DebugToolbarExtension
+    toolbar = DebugToolbarExtension(app)

celery_config.py

@@ -0,0 +1,17 @@
+from celery import Celery
+
+celery = Celery("procrustes_server", broker="redis://redis:6379/0", backend="redis://redis:6379/0")
+
+celery.conf.update(
+    worker_max_tasks_per_child=1000,
+    worker_prefetch_multiplier=1,
+    task_acks_late=True,
+    task_reject_on_worker_lost=True,
+    broker_pool_limit=None,
+    broker_connection_timeout=30,
+    result_expires=3600,  # Results expire after 1 hour
+    task_track_started=True,
+    task_time_limit=300,  # 5 minutes
+    task_soft_time_limit=240,  # 4 minutes
+    worker_concurrency=4,  # Number of worker processes per Celery worker
+)

docker-compose.yml

@@ -0,0 +1,96 @@
+version: '3.8'
+
+services:
+  web:
+    build: .
+    command: gunicorn --config gunicorn_config.py app:app --reload
+    expose:
+      - "8008"
+    volumes:
+      - .:/app
+      - upload_data:/app/uploads
+    depends_on:
+      - redis
+    environment:
+      - FLASK_ENV=development
+      - REDIS_URL=redis://redis:6379/0
+    deploy:
+      replicas: 2
+      resources:
+        limits:
+          cpus: '1'
+          memory: 2G
+    restart: unless-stopped
+
+  redis:
+    image: redis:7-alpine
+    command: redis-server --appendonly yes
+    volumes:
+      - redis_data:/data
+    ports:
+      - "6379:6379"
+    deploy:
+      resources:
+        limits:
+          cpus: '0.5'
+          memory: 1G
+    restart: unless-stopped
+
+  celery_worker:
+    build: .
+    command: celery -A app.celery worker --loglevel=info
+    volumes:
+      - .:/app
+      - upload_data:/app/uploads
+    depends_on:
+      - redis
+    environment:
+      - REDIS_URL=redis://redis:6379/0
+    deploy:
+      replicas: 4
+      resources:
+        limits:
+          cpus: '1'
+          memory: 2G
+    restart: unless-stopped
+
+  celery_flower:
+    build: .
+    command: celery -A app.celery flower
+    ports:
+      - "5555:5555"
+    volumes:
+      - .:/app
+      - flower_data:/app/flower
+    depends_on:
+      - redis
+      - celery_worker
+    environment:
+      - REDIS_URL=redis://redis:6379/0
+    deploy:
+      resources:
+        limits:
+          cpus: '0.5'
+          memory: 512M
+    restart: unless-stopped
+
+  nginx:
+    image: nginx:alpine
+    ports:
+      - "8008:8008"
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf:ro
+      - .:/app:ro
+    depends_on:
+      - web
+    deploy:
+      resources:
+        limits:
+          cpus: '0.5'
+          memory: 512M
+    restart: unless-stopped
+
+volumes:
+  redis_data:
+  upload_data:
+  flower_data:

gunicorn_config.py

@@ -0,0 +1,55 @@
+import multiprocessing
+import os
+
+# Number of worker processes - adjust based on CPU cores
+workers = multiprocessing.cpu_count() * 2 + 1
+
+# Number of threads per worker
+threads = 4
+
+# Maximum number of pending connections
+backlog = 2048
+
+# Maximum number of requests a worker will process before restarting
+max_requests = 10000
+max_requests_jitter = 50
+
+# Timeout for worker processes (5 minutes)
+timeout = 300
+
+# Keep-alive timeout
+keepalive = 5
+
+# Log settings
+loglevel = "info"
+accesslog = "-"
+errorlog = "-"
+access_log_format = '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"'
+
+# Bind address - use HF_PORT for Hugging Face deployment
+port = os.getenv('PORT', '8008')  # HF uses PORT env var
+bind = f"0.0.0.0:{port}"
+
+# Worker class
+worker_class = "gevent"
+
+# Process name
+proc_name = "selector_server"
+
+# Preload app for faster worker spawning
+preload_app = True
+
+# Graceful timeout
+graceful_timeout = 30
+
+# Server mechanics
+daemon = False
+pidfile = None
+umask = 0
+user = None
+group = None
+tmp_upload_dir = None
+
+# Logging
+capture_output = True
+enable_stdio_inheritance = False

md_files/about.md

@@ -0,0 +1,22 @@
+## Motivation
+
+Selecting diverse and representative subsets is crucial for the data-driven models and machine learning applications in many science and engineering disciplines, especially for molecular design and drug discovery. Motivated by this, we develop the Selector package, a free and open-source Python library for selecting diverse subsets.
+
+The `Selector` library implements a range of existing algorithms for subset sampling based on the distance between and similarity of samples, as well as tools based on spatial partitioning. In addition, it includes seven diversity measures for quantifying the diversity of a given set. We also implemented various mathematical formulations to convert similarities into dissimilarities.
+
+## `Selector` Library
+
+Selector is a free, open-source, and cross-platform Python library designed to help you effortlessly identify the most diverse subset of molecules from your dataset. Please use the following citation in any publication using Selector library:
+
+## Citation
+Please use the following citation in any publication using the `Selector` library:
+
+**To be added**
+
+## More Information
+
+For more information about the Selector library, please visit our [GitHub repository](https://github.com/qcdevs/selector) and documentation at [https://selector.qcdevs.org](https://selector.qcdevs.org).
+
+## Acknowledgments
+
+This webserver is supported by the DRI EDIA Champions Pilot Program. We are grateful to the [Digital Research Alliance](https://alliancecan.ca/) for providing the computing resources.

md_files/contacts.md

@@ -0,0 +1,2 @@
+The `Selector` source code is hosted on [GitHub](https://github.com/theochem/Selector) and is released under the [GNU General Public License v3.0](https://github.com/theochem/Selector/blob/master/LICENSE). We welcome any contributions to the Selector library in accordance with our Code of Conduct; please see our [Contributing Guidelines](https://qcdevs.org/guidelines/QCDevsCodeOfConduct/). Please report any issues you encounter while using Selector library on [GitHub Issues](https://github.com/theochem/Selector/issues). For further information and inquiries, please contact us at [[email protected]](mailto:[email protected]).
+

nginx.conf

@@ -0,0 +1,62 @@
+user nginx;
+worker_processes auto;
+error_log /var/log/nginx/error.log warn;
+pid /var/run/nginx.pid;
+
+events {
+    worker_connections 4096;
+    use epoll;
+    multi_accept on;
+}
+
+http {
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
+                    '$status $body_bytes_sent "$http_referer" '
+                    '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log /var/log/nginx/access.log main;
+
+    sendfile on;
+    tcp_nopush on;
+    tcp_nodelay on;
+    keepalive_timeout 65;
+    types_hash_max_size 2048;
+    client_max_body_size 32M;
+
+    # Gzip compression
+    gzip on;
+    gzip_vary on;
+    gzip_proxied any;
+    gzip_comp_level 6;
+    gzip_types text/plain text/css text/xml application/json application/javascript application/xml+rss application/atom+xml image/svg+xml;
+
+    upstream selector_backend {
+        server web:8008;
+    }
+
+    server {
+        listen 8008;
+        server_name localhost;
+
+        location / {
+            proxy_pass http://selector_backend;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+        }
+
+        location /flower/ {
+            rewrite ^/flower/(.*)$ /$1 break;
+            proxy_pass http://celery_flower:5555;
+            proxy_set_header Host $host;
+            proxy_redirect off;
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+        }
+    }
+}

prometheus.yml

@@ -0,0 +1,20 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: 'prometheus'
+    static_configs:
+      - targets: ['localhost:9090']
+
+  - job_name: 'flask'
+    static_configs:
+      - targets: ['web:8000']
+
+  - job_name: 'redis'
+    static_configs:
+      - targets: ['redis-exporter:9121']
+
+  - job_name: 'flower'
+    static_configs:
+      - targets: ['flower:5555']

requirements.txt

@@ -0,0 +1,17 @@
+flask>=3.0.0
+qc-selector>=0.0.3
+numpy>=2.0.0
+pandas>=2.1.4
+openpyxl>=3.0.9
+werkzeug>=3.0.0
+gunicorn>=22.0.0
+gevent>=23.9.1
+redis>=5.0.1
+celery>=5.3.6
+flower>=2.0.1
+orjson>=3.10.12
+flask_status>=1.0.1
+markdown>=3.5.1
+Pygments>=2.17.2
+matplotlib>=3.8.0
+scikit-learn>=1.5.0

templates/index.html

@@ -0,0 +1,998 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Selector Web Server</title>
+    <link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet">
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/font/bootstrap-icons.css">
+    <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600&display=swap" rel="stylesheet">
+    <style>
+        :root {
+            --sidebar-width: 280px;
+            --primary-color: #2563eb;
+            --sidebar-bg: #f8fafc;
+            --header-bg: #f1f5f9;
+            --border-color: #e2e8f0;
+            --text-primary: #1e293b;
+            --text-secondary: #64748b;
+        }
+
+        body {
+            font-family: 'Inter', -apple-system, BlinkMacSystemFont, sans-serif;
+            color: var(--text-primary);
+            line-height: 1.6;
+        }
+
+        .wrapper {
+            display: flex;
+            width: 100%;
+            align-items: stretch;
+        }
+
+        #sidebar {
+            min-width: var(--sidebar-width);
+            max-width: var(--sidebar-width);
+            min-height: 100vh;
+            transition: all 0.3s ease-out;
+            background-color: var(--sidebar-bg);
+            border-right: 1px solid var(--border-color);
+        }
+
+        #sidebar.active {
+            margin-left: calc(-1 * var(--sidebar-width));
+        }
+
+        #content {
+            width: 100%;
+            padding: 2rem;
+            min-height: 100vh;
+            margin-left: 0;
+            transition: all 0.3s ease-out;
+            background-color: white;
+            padding-bottom: 4rem;
+        }
+
+        .sidebar-header {
+            padding: 1.5rem;
+            background: var(--header-bg);
+            display: flex;
+            align-items: center;
+            justify-content: space-between;
+            border-bottom: 1px solid var(--border-color);
+        }
+
+        #sidebarCollapse, #showSidebarBtn {
+            background: transparent;
+            border: none;
+            padding: 0.5rem;
+            color: var(--text-primary);
+            border-radius: 0.375rem;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            cursor: pointer;
+            transition: all 0.2s;
+        }
+
+        #sidebarCollapse:hover, #showSidebarBtn:hover {
+            background: rgba(0, 0, 0, 0.05);
+            transform: translateX(2px);
+        }
+
+        #sidebarCollapse i {
+            transition: transform 0.3s ease;
+        }
+
+        #sidebar.active #sidebarCollapse i {
+            transform: rotate(180deg);
+        }
+
+        #showSidebarBtn {
+            position: fixed;
+            left: 1.5rem;
+            top: 1.5rem;
+            z-index: 1000;
+            display: none;
+            background-color: white;
+            border: 1px solid var(--border-color);
+            box-shadow: 0 1px 3px rgba(0,0,0,0.1);
+        }
+
+        #showSidebarBtn.show {
+            display: flex;
+        }
+
+        .sidebar-title {
+            margin: 0;
+            font-weight: 600;
+            color: var(--text-primary);
+            font-size: 1.125rem;
+        }
+
+        .components {
+            padding: 1.5rem 0;
+        }
+
+        .components a {
+            padding: 0.75rem 1.5rem;
+            display: flex;
+            align-items: center;
+            color: var(--text-secondary);
+            text-decoration: none;
+            transition: all 0.2s;
+            font-weight: 500;
+        }
+
+        .components a i {
+            margin-right: 0.75rem;
+            font-size: 1.25rem;
+        }
+
+        .components a:hover {
+            background: var(--header-bg);
+            color: var(--text-primary);
+        }
+
+        .components a.active {
+            background: var(--header-bg);
+            color: var(--primary-color);
+            border-right: 3px solid var(--primary-color);
+        }
+
+        .card {
+            border: 1px solid var(--border-color);
+            border-radius: 0.5rem;
+            box-shadow: 0 1px 3px rgba(0,0,0,0.1);
+        }
+
+        .card-body {
+            padding: 1.5rem;
+        }
+
+        .form-label {
+            font-weight: 500;
+            color: var(--text-primary);
+            margin-bottom: 0.5rem;
+        }
+
+        .form-control, .form-select {
+            border-color: var(--border-color);
+            border-radius: 0.375rem;
+            padding: 0.625rem;
+        }
+
+        .form-control:focus, .form-select:focus {
+            border-color: var(--primary-color);
+            box-shadow: 0 0 0 2px rgba(37, 99, 235, 0.1);
+        }
+
+        .btn-primary {
+            background-color: var(--primary-color);
+            border: none;
+            padding: 0.625rem 1.25rem;
+            font-weight: 500;
+        }
+
+        .btn-primary:hover {
+            background-color: #1d4ed8;
+        }
+
+        .btn-success, .btn-info {
+            font-weight: 500;
+            padding: 0.625rem 1.25rem;
+        }
+
+        .loading {
+            display: none;
+            align-items: center;
+            gap: 0.75rem;
+            padding: 1rem;
+            background-color: var(--sidebar-bg);
+            border-radius: 0.375rem;
+        }
+
+        .result-section {
+            display: none;
+            margin-top: 2rem;
+        }
+
+        .bg-light {
+            background-color: var(--sidebar-bg) !important;
+            border-radius: 0.375rem;
+        }
+
+        h2 {
+            font-weight: 600;
+            margin-bottom: 1.5rem;
+            color: var(--text-primary);
+        }
+
+        h5, h6 {
+            font-weight: 600;
+            color: var(--text-primary);
+        }
+
+        .footer {
+            position: fixed;
+            bottom: 0;
+            width: 100%;
+            background-color: var(--header-bg);
+            border-top: 1px solid var(--border-color);
+            padding: 1rem;
+            display: flex;
+            justify-content: space-between;
+            align-items: center;
+            z-index: 1000;
+        }
+
+        .footer .copyright {
+            color: var(--text-secondary);
+            font-size: 0.875rem;
+        }
+
+        .footer .status {
+            display: flex;
+            align-items: center;
+            gap: 0.75rem;
+            color: var(--text-secondary);
+            font-size: 0.875rem;
+        }
+
+        .footer .status .status-indicator {
+            width: 8px;
+            height: 8px;
+            border-radius: 50%;
+        }
+
+        .footer .status .status-details {
+            display: none;
+            position: absolute;
+            bottom: 100%;
+            right: 1rem;
+            background: white;
+            border: 1px solid var(--border-color);
+            border-radius: 0.5rem;
+            padding: 1rem;
+            box-shadow: 0 -4px 6px -1px rgba(0, 0, 0, 0.1);
+            min-width: 200px;
+        }
+
+        .footer .status:hover .status-details {
+            display: block;
+        }
+
+        .status-details div {
+            margin-bottom: 0.5rem;
+        }
+
+        .status-details .component {
+            display: flex;
+            align-items: center;
+            gap: 0.5rem;
+        }
+
+        .status-details .component-indicator {
+            width: 6px;
+            height: 6px;
+            border-radius: 50%;
+        }
+
+        .status-details .resources {
+            margin-top: 0.75rem;
+            padding-top: 0.75rem;
+            border-top: 1px solid var(--border-color);
+        }
+
+        /* Markdown content styling */
+        .section {
+            padding: 2rem;
+        }
+
+        .section table {
+            border-collapse: collapse;
+            margin: 1rem 0;
+            width: 100%;
+        }
+
+        .section table th,
+        .section table td {
+            border: 1px solid var(--border-color);
+            padding: 0.75rem;
+            text-align: left;
+        }
+
+        .section table th {
+            background-color: var(--header-bg);
+            font-weight: 600;
+        }
+
+        .section table tr:nth-child(even) {
+            background-color: var(--header-bg);
+        }
+
+        .section h1,
+        .section h2,
+        .section h3 {
+            color: var(--text-primary);
+            margin-top: 2rem;
+            margin-bottom: 1rem;
+        }
+
+        .section h1 {
+            font-size: 2rem;
+            border-bottom: 2px solid var(--border-color);
+            padding-bottom: 0.5rem;
+        }
+
+        .section h2 {
+            font-size: 1.5rem;
+        }
+
+        .section h3 {
+            font-size: 1.25rem;
+        }
+
+        .section p {
+            margin-bottom: 1rem;
+            line-height: 1.6;
+        }
+
+        .section ul,
+        .section ol {
+            margin-bottom: 1rem;
+            padding-left: 2rem;
+        }
+
+        .section li {
+            margin-bottom: 0.5rem;
+        }
+
+        .section code {
+            background-color: var(--header-bg);
+            padding: 0.2rem 0.4rem;
+            border-radius: 0.25rem;
+            font-family: monospace;
+        }
+
+        .section pre {
+            background-color: var(--header-bg);
+            padding: 1rem;
+            border-radius: 0.5rem;
+            overflow-x: auto;
+            margin: 1rem 0;
+        }
+
+        .section a {
+            color: var(--primary-color);
+            text-decoration: none;
+        }
+
+        .section a:hover {
+            text-decoration: underline;
+        }
+
+        /* Math equations */
+        .section .MathJax_Display {
+            overflow-x: auto;
+            max-width: 100%;
+            margin: 1em 0;
+        }
+
+        .section .MathJax {
+            font-size: 1.1em !important;
+        }
+
+        .section .math-block {
+            overflow-x: auto;
+            margin: 1em 0;
+            text-align: center;
+        }
+    </style>
+    <!-- MathJax Configuration -->
+        <script type="text/x-mathjax-config">
+        MathJax.Hub.Config({
+            tex2jax: {
+        inlineMath: [['$','$'], ['\\(','\\)']],
+        displayMath: [['$$','$$'], ['\\[','\\]']],
+        processEscapes: true,
+        processEnvironments: true
+        },
+        displayAlign: 'center',
+        "HTML-CSS": {
+            styles: {'.MathJax_Display': {"margin": 0}},
+            linebreaks: { automatic: true }
+        }
+        });
+    </script>
+
+    <!-- MathJax Loading -->
+    <script type="text/javascript" src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+</head>
+<body>
+    <button type="button" id="showSidebarBtn" class="btn">
+        <i class="bi bi-chevron-right"></i>
+    </button>
+
+    <div class="wrapper">
+        <!-- Sidebar -->
+        <nav id="sidebar">
+            <div class="sidebar-header">
+                <h5 class="sidebar-title">Selector Server</h5>
+                <button type="button" id="sidebarCollapse" class="btn">
+                    <i class="bi bi-chevron-left"></i>
+                </button>
+            </div>
+            <ul class="nav flex-column">
+                <li class="nav-item">
+                    <a href="#" class="nav-link active" data-section="selection">
+                        <i class="bi bi-grid-3x3-gap me-2"></i>Subset Selection
+                    </a>
+                </li>
+                <li class="nav-item">
+                    <a href="#" class="nav-link" data-section="diversity">
+                        <i class="bi bi-graph-up me-2"></i>Diversity Calculation
+                    </a>
+                </li>
+                <li class="nav-item">
+                    <a href="#" class="nav-link" data-section="about">
+                        <i class="bi bi-info-circle me-2"></i>About
+                    </a>
+                </li>
+                <li class="nav-item">
+                    <a href="#" class="nav-link" data-section="contact">
+                        <i class="bi bi-envelope me-2"></i>Contact
+                    </a>
+                </li>
+            </ul>
+        </nav>
+
+        <!-- Page Content -->
+        <div id="content">
+            <!-- Selection Section -->
+            <div id="selection" class="section">
+                <h2>Subset Selection</h2>
+                <div class="card">
+                    <div class="card-body">
+                        <form id="selectionForm" enctype="multipart/form-data">
+                            <div class="mb-3">
+                                <label for="file" class="form-label">Upload Feature Matrix</label>
+                                <input type="file" class="form-control" id="file" name="file" accept=".txt,.npz,.xlsx,.xls">
+                                <small class="text-muted">Supported formats: .txt, .npz, .xlsx, .xls (max 32MB)</small>
+                            </div>
+
+                            <div class="mb-3">
+                                <label for="size" class="form-label">Subset Size</label>
+                                <input type="number" class="form-control" id="size" name="size" min="1" step="1" required>
+                                <small class="text-muted">Number of data points to select (integer)</small>
+                            </div>
+
+                            <div class="mb-3">
+                                <label for="distance_metric" class="form-label">Distance Function</label>
+                                <select class="form-select" id="distance_metric" name="distance_metric">
+                                    <option value="euclidean">euclidean</option>
+                                    <option value="cityblock">cityblock</option>
+                                    <option value="cosine">cosine</option>
+                                    <option value="l1">l1</option>
+                                    <option value="l2">l2</option>
+                                    <option value="manhattan">manhattan</option>
+                                    <option value="braycurtis">braycurtis</option>
+                                    <option value="canberra">canberra</option>
+                                    <option value="chebyshev">chebyshev</option>
+                                    <option value="correlation">correlation</option>
+                                    <option value="dice">dice</option>
+                                    <option value="hamming">hamming</option>
+                                    <option value="jaccard">jaccard</option>
+                                    <option value="kulsinski">kulsinski</option>
+                                    <option value="mahalanobis">mahalanobis</option>
+                                    <option value="minkowski">minkowski</option>
+                                    <option value="rogerstanimoto">rogerstanimoto</option>
+                                    <option value="russellrao">russellrao</option>
+                                    <option value="seuclidean">seuclidean</option>
+                                    <option value="sokalmichener">sokalmichener</option>
+                                    <option value="sokalsneath">sokalsneath</option>
+                                    <option value="sqeuclidean">sqeuclidean</option>
+                                    <option value="yule">yule</option>
+                                </select>
+                                <small class="text-muted">Distance metric from scikit-learn's pairwise_distances (optional)</small>
+                            </div>
+
+                            <div class="mb-3">
+                                <label for="algorithm" class="form-label">Selection Method</label>
+                                <select class="form-select" id="algorithm" name="algorithm">
+                                    <optgroup label="Distance-based Methods">
+                                        <option value="MaxMin">MaxMin</option>
+                                        <option value="MaxSum">MaxSum</option>
+                                        <option value="OptiSim">OptiSim</option>
+                                        <option value="DISE">DISE</option>
+                                    </optgroup>
+                                    <optgroup label="Partition-based Methods">
+                                        <option value="GridPartition">Grid Partition</option>
+                                    </optgroup>
+                                    <optgroup label="Similarity-based Methods">
+                                        <option value="NSimilarity">NSimilarity</option>
+                                    </optgroup>
+                                </select>
+                            </div>
+
+                            <div class="mb-3" id="nbins_axis_container" style="display: none;">
+                                <label for="nbins_axis" class="form-label">Number of Bins per Axis</label>
+                                <input type="number" class="form-control" id="nbins_axis" name="nbins_axis" min="1" value="5">
+                                <small class="form-text text-muted">Number of bins to partition each axis into</small>
+                            </div>
+
+                            <div class="mb-3">
+                                <label for="parameters" class="form-label">Additional Parameters (JSON format)</label>
+                                <textarea class="form-control font-monospace" id="parameters" name="parameters" rows="10"></textarea>
+                            </div>
+
+                            <div class="d-grid gap-2">
+                                <button type="submit" class="btn btn-primary">
+                                    <i class="bi bi-play-fill me-2"></i>Run Subset Selection
+                                </button>
+                            </div>
+                        </form>
+                    </div>
+                </div>
+
+                <div id="resultSection" class="mt-4" style="display: none;">
+                    <div class="card">
+                        <div class="card-body">
+                            <h5 class="card-title">Results</h5>
+                            <div id="warnings" class="alert alert-warning" style="display: none;"></div>
+                            <div id="error" class="alert alert-danger" style="display: none;"></div>
+
+                            <div id="success-content" style="display: none;">
+                                <h4>Selected Indices:</h4>
+                                <pre id="indices"></pre>
+                                <div class="mb-3">
+                                    <label for="download-format" class="form-label">Download Format</label>
+                                    <select class="form-select" id="download-format">
+                                        <option value="txt">TXT</option>
+                                        <option value="npz">NPZ</option>
+                                        <option value="xlsx">Excel</option>
+                                    </select>
+                                </div>
+                                <button class="btn btn-primary" onclick="downloadIndices()">Download Results</button>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+            </div>
+
+            <!-- Diversity Section -->
+            <div id="diversity" class="section" style="display: none;">
+                <h2>Diversity Calculation</h2>
+                <div class="container mt-4">
+                    <form id="diversity-form" enctype="multipart/form-data">
+                        <div class="mb-3">
+                            <label for="feature_subset" class="form-label">Upload Feature Subset</label>
+                            <input type="file" class="form-control" id="feature_subset" name="feature_subset" accept=".csv,.txt,.xlsx,.xls,.npz">
+                            <small class="text-muted">Upload your feature subset matrix (supported formats: CSV, TXT, XLSX, NPZ)</small>
+                        </div>
+
+                        <div class="mb-3">
+                            <label for="features" class="form-label">Upload Features (Optional)</label>
+                            <input type="file" class="form-control" id="features" name="features" accept=".csv,.txt,.xlsx,.xls,.npz">
+                            <small class="text-muted">Upload your features matrix. It's only for hypersphere overlaping algorithm.</small>
+                        </div>
+
+                        <div class="mb-3">
+                            <label for="div_type" class="form-label">Diversity Type</label>
+                            <select class="form-select" id="div_type" name="div_type">
+                                <option value="logdet">Log Determinant</option>
+                                <option value="shannon_entropy">Shannon Entropy</option>
+                                <option value="explicit_diversity_index">Explicit Diversity Index</option>
+                                <option value="wdud">WDUD</option>
+                                <option value="hypersphere_overlap">Hypersphere Overlap</option>
+                                <option value="gini_coefficient">Gini Coefficient</option>
+                            </select>
+                        </div>
+
+                        <div class="mb-3">
+                            <label for="div_parameters" class="form-label">Additional Parameters (JSON format)</label>
+                            <textarea class="form-control font-monospace" id="div_parameters" name="div_parameters" rows="5"></textarea>
+                            <small class="text-muted">Specify normalize, truncation, and cs parameters in JSON format</small>
+                        </div>
+
+                        <div class="d-grid gap-2">
+                            <button type="submit" class="btn btn-primary">Calculate Diversity</button>
+                        </div>
+
+                        <!-- Results section -->
+                        <div id="diversity-results" class="mt-4" style="display: none;">
+                            <h3>Results</h3>
+                            <div class="card">
+                                <div class="card-body">
+                                    <h5 class="card-title">Diversity Score</h5>
+                                    <p class="card-text" id="diversity-score"></p>
+                                </div>
+                            </div>
+                        </div>
+                    </form>
+                </div>
+            </div>
+
+            <!-- About Section -->
+            <div id="about" class="section" style="display: none;">
+                <h2>About Selector</h2>
+                <div id="about-content">Loading...</div>
+            </div>
+
+            <!-- Contact Section -->
+            <div id="contact" class="section" style="display: none;">
+                <h2>Contact Information</h2>
+                <div id="contact-content">Loading...</div>
+            </div>
+        </div>
+    </div>
+
+    <!-- Footer -->
+    <div class="footer">
+        <div class="copyright">
+            &copy; Copyright 2017-2024, The QC-Devs Community.
+        </div>
+        <div class="status">
+            <div class="status-indicator"></div>
+            <span>Server Status: </span>
+            <span id="serverStatus">Loading...</span>
+            <div class="status-details">
+                <div><strong>Components:</strong></div>
+                <div class="component">
+                    <div class="component-indicator"></div>
+                    <span>Flask</span>
+                </div>
+                <div class="component">
+                    <div class="component-indicator"></div>
+                    <span>Celery</span>
+                </div>
+                <div class="component">
+                    <div class="component-indicator"></div>
+                    <span>Redis</span>
+                </div>
+                <div class="resources">
+                    <div><strong>Resources:</strong></div>
+                    <div>CPU: <span id="cpuStatus">-</span>%</div>
+                    <div>Memory: <span id="memoryStatus">-</span>%</div>
+                    <div>Disk: <span id="diskStatus">-</span>%</div>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js"></script>
+    <script>
+        let currentIndices = null;  // Store the current indices globally
+
+        // Sidebar toggle functionality
+        const sidebar = document.getElementById('sidebar');
+        const content = document.getElementById('content');
+        const sidebarCollapse = document.getElementById('sidebarCollapse');
+        const showSidebarBtn = document.getElementById('showSidebarBtn');
+
+        function toggleSidebar() {
+            sidebar.classList.toggle('active');
+            content.classList.toggle('active');
+            showSidebarBtn.classList.toggle('show', sidebar.classList.contains('active'));
+        }
+
+        sidebarCollapse.addEventListener('click', toggleSidebar);
+        showSidebarBtn.addEventListener('click', toggleSidebar);
+
+        function showSection(sectionId) {
+            // Hide all sections
+            document.querySelectorAll('.section').forEach(section => {
+                section.style.display = 'none';
+            });
+
+            // Remove active class from all nav links
+            document.querySelectorAll('.nav-link').forEach(link => {
+                link.classList.remove('active');
+            });
+
+            // Show selected section
+            const section = document.getElementById(sectionId);
+            if (section) {
+                section.style.display = 'block';
+                // Add active class to corresponding nav link
+                const navLink = document.querySelector(`.nav-link[data-section="${sectionId}"]`);
+                if (navLink) {
+                    navLink.classList.add('active');
+                }
+            }
+        }
+
+        // Handle navigation clicks
+        document.querySelectorAll('.nav-link').forEach(link => {
+            link.addEventListener('click', (e) => {
+                e.preventDefault();
+                const sectionId = link.getAttribute('data-section');
+                showSection(sectionId);
+            });
+        });
+
+        // Show selection section by default
+        document.addEventListener('DOMContentLoaded', function() {
+            showSection('selection');
+            loadMarkdownContent();
+            checkServerStatus();
+            setInterval(checkServerStatus, 30000);
+        });
+
+        async function updateDefaultParams() {
+            const algorithm = document.getElementById('algorithm').value;
+            const parametersTextarea = document.getElementById('parameters');
+            const nbinsInput = document.getElementById('nbins_axis');
+
+            try {
+                const response = await fetch(`/get_default_params/${algorithm}`);
+                const data = await response.json();
+
+                if (response.ok) {
+                    // Get base parameters
+                    const params = data;
+
+                    // Add nbins_axis for GridPartition
+                    if (algorithm === 'GridPartition') {
+                        params.nbins_axis = parseInt(nbinsInput.value) || 5;
+                    }
+
+                    // Format the parameters nicely
+                    parametersTextarea.value = JSON.stringify(params, null, 2);
+                } else {
+                    console.error('Error getting default parameters:', data.error);
+                    parametersTextarea.value = '{}';
+                }
+            } catch (error) {
+                console.error('Error:', error);
+                parametersTextarea.value = '{}';
+            }
+        }
+
+        // Update parameters when algorithm changes
+        document.getElementById('algorithm').addEventListener('change', function() {
+            const algorithm = this.value;
+            const nbinsContainer = document.getElementById('nbins_axis_container');
+
+            // Show/hide nbins_axis input based on algorithm
+            if (algorithm === 'GridPartition') {
+                nbinsContainer.style.display = 'block';
+            } else {
+                nbinsContainer.style.display = 'none';
+            }
+
+            updateDefaultParams();
+        });
+
+        // Update parameters when nbins_axis changes
+        document.getElementById('nbins_axis').addEventListener('input', function() {
+            if (document.getElementById('algorithm').value === 'GridPartition') {
+                const params = JSON.parse(document.getElementById('parameters').value || '{}');
+                params.nbins_axis = parseInt(this.value) || 5;
+                document.getElementById('parameters').value = JSON.stringify(params, null, 2);
+            }
+        });
+
+        // Update parameters when page loads
+        document.addEventListener('DOMContentLoaded', function() {
+            updateDefaultParams();
+            // Also trigger algorithm change to show/hide nbins_axis input
+            const algorithmSelect = document.getElementById('algorithm');
+            algorithmSelect.dispatchEvent(new Event('change'));
+        });
+
+        async function downloadIndices() {
+            if (!currentIndices) {
+                console.error('No indices available for download');
+                return;
+            }
+
+            const format = document.getElementById('download-format').value;
+            const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+
+            try {
+                const response = await fetch('/download', {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json',
+                    },
+                    body: JSON.stringify({
+                        indices: currentIndices,
+                        format: format,
+                        timestamp: timestamp
+                    })
+                });
+
+                if (response.ok) {
+                    // Create a download link
+                    const blob = await response.blob();
+                    const url = window.URL.createObjectURL(blob);
+                    const a = document.createElement('a');
+                    a.href = url;
+                    a.download = `selected_indices_${timestamp}.${format}`;
+                    document.body.appendChild(a);
+                    a.click();
+                    window.URL.revokeObjectURL(url);
+                    a.remove();
+                } else {
+                    const error = await response.json();
+                    console.error('Download failed:', error);
+                    alert('Failed to download results: ' + (error.error || 'Unknown error'));
+                }
+            } catch (error) {
+                console.error('Download error:', error);
+                alert('Error downloading results: ' + error.message);
+            }
+        }
+
+        document.getElementById('selectionForm').addEventListener('submit', async (e) => {
+            e.preventDefault();
+
+            const formData = new FormData(e.target);
+            const resultSection = document.getElementById('resultSection');
+            const warningsDiv = document.getElementById('warnings');
+            const errorDiv = document.getElementById('error');
+            const successContent = document.getElementById('success-content');
+            const indicesDiv = document.getElementById('indices');
+
+            // Validate size is an integer
+            const size = parseInt(formData.get('size'));
+            if (isNaN(size) || size < 1 || size !== parseFloat(formData.get('size'))) {
+                errorDiv.textContent = 'Size must be a positive integer';
+                resultSection.style.display = 'block';
+                return;
+            }
+
+            try {
+                const response = await fetch('/upload_selection', {
+                    method: 'POST',
+                    body: formData
+                });
+
+                const data = await response.json();
+                console.log('Response data:', data);
+
+                // Reset display
+                resultSection.style.display = 'block';
+                warningsDiv.style.display = 'none';
+                errorDiv.style.display = 'none';
+                successContent.style.display = 'none';
+
+                if (response.ok && data.success) {
+                    // Store the indices globally
+                    currentIndices = data.indices;
+
+                    // Show warnings if any
+                    if (data.warnings && data.warnings.length > 0) {
+                        warningsDiv.textContent = 'Warnings: ' + data.warnings.join(', ');
+                        warningsDiv.style.display = 'block';
+                    }
+
+                    // Show results
+                    indicesDiv.textContent = JSON.stringify(data.indices, null, 2);
+                    successContent.style.display = 'block';
+                } else {
+                    errorDiv.textContent = data.error || 'Failed to process selection';
+                    errorDiv.style.display = 'block';
+                }
+            } catch (error) {
+                console.error('Error:', error);
+                errorDiv.textContent = 'Error processing request: ' + error.message;
+                errorDiv.style.display = 'block';
+            }
+        });
+
+        // Server status check
+        async function checkServerStatus() {
+            try {
+                const response = await fetch('/status');
+                const data = await response.json();
+                const statusElem = document.getElementById('serverStatus');
+                const indicator = document.querySelector('.status-indicator');
+
+                // Update main status
+                statusElem.textContent = data.status === 'ok' ? 'Running' :
+                                       data.status === 'degraded' ? 'Degraded' : 'Error';
+
+                // Update indicator color
+                indicator.style.backgroundColor =
+                    data.status === 'ok' ? '#22c55e' :      // green
+                    data.status === 'degraded' ? '#f59e0b' : // orange
+                    '#ef4444';                               // red
+
+                // Update components
+                const components = document.querySelectorAll('.component');
+                Object.entries(data.components).forEach((component, index) => {
+                    const [name, status] = component;
+                    components[index].querySelector('.component-indicator').style.backgroundColor =
+                        status ? '#22c55e' : '#ef4444';
+                });
+            } catch (error) {
+                const statusElem = document.getElementById('serverStatus');
+                const indicator = document.querySelector('.status-indicator');
+                statusElem.textContent = 'Offline';
+                indicator.style.backgroundColor = '#ef4444';  // red
+
+                // Set all components to red when offline
+                const components = document.querySelectorAll('.component');
+                components.forEach(comp => {
+                    comp.querySelector('.component-indicator').style.backgroundColor = '#ef4444';
+                });
+            }
+        }
+
+        // Load markdown content and add MathJax support
+        async function loadMarkdownContent() {
+            try {
+                // Load about content
+                const aboutResponse = await fetch('/md/about.md');
+                const aboutData = await aboutResponse.json();
+                document.getElementById('about-content').innerHTML = aboutData.html;
+
+                // Load contact content
+                const contactResponse = await fetch('/md/contacts.md');
+                const contactData = await contactResponse.json();
+                document.getElementById('contact-content').innerHTML = contactData.html;
+
+                // Retypeset math
+                if (window.MathJax && window.MathJax.Hub) {
+                    window.MathJax.Hub.Queue(["Typeset", window.MathJax.Hub]);
+                }
+            } catch (error) {
+                console.error('Error loading markdown content:', error);
+            }
+        }
+
+        document.getElementById('diversity-form').addEventListener('submit', async function(e) {
+            e.preventDefault();
+
+            const formData = new FormData(this);
+            const parameters = document.getElementById('div_parameters').value;
+
+            try {
+                // Parse and add JSON parameters
+                const params = JSON.parse(parameters || '{}');
+                formData.set('div_parameters', JSON.stringify(params));
+
+                const response = await fetch('/calculate_diversity', {
+                    method: 'POST',
+                    body: formData
+                });
+
+                const result = await response.json();
+
+                if (response.ok) {
+                    // Show results
+                    document.getElementById('diversity-results').style.display = 'block';
+                    document.getElementById('diversity-score').textContent = result.diversity_score;
+                } else {
+                    alert('Error: ' + result.error);
+                }
+            } catch (error) {
+                console.error('Error:', error);
+                alert('Error calculating diversity: ' + error.message);
+            }
+        });
+
+        // Set default parameters when diversity type changes
+        document.getElementById('div_type').addEventListener('change', function() {
+            const defaultParams = {
+                normalize: false,
+                truncation: false,
+                cs: null
+            };
+            document.getElementById('div_parameters').value = JSON.stringify(defaultParams, null, 2);
+        });
+
+        // Initialize default parameters
+        document.addEventListener('DOMContentLoaded', function() {
+            const defaultParams = {
+                normalize: false,
+                truncation: false,
+                cs: null
+            };
+            document.getElementById('div_parameters').value = JSON.stringify(defaultParams, null, 2);
+        });
+    </script>
+</body>
+</html>

tests/generate_test_data.py

@@ -0,0 +1,8 @@
+import numpy as np
+
+# Create a simple 3x3 matrix
+matrix = np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]])
+
+# Save two identical copies
+np.savez("matrix1.npz", arr_0=matrix)
+np.savez("matrix2.npz", arr_0=matrix)