feat: init

Dockerfile

@@ -0,0 +1,37 @@
+FROM debian:bookworm-slim
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    aria2 \
+    nginx \
+    curl \
+    unzip \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN aria2c -c -x16 https://github.com/MZWNET/actions/releases/download/nitro-v0.3.18/nitro-0.3.18-linux-amd64-intel-mkl-avx2.zip \
+    && unzip nitro-0.3.18-linux-amd64-intel-mkl-avx2.zip \
+    && mv nitro-0.3.18-linux-amd64-intel-mkl-avx2/nitro /app/nitro \
+    && chmod +x /app/nitro \
+    && rm -rf nitro-0.3.18-linux-amd64-intel-mkl-avx2.zip nitro-0.3.18-linux-amd64-intel-mkl-avx2
+
+COPY nginx.conf /etc/nginx/nginx.conf --chown=user
+
+RUN echo '#!/bin/sh\n\
+/app/nitro &\n\
+nginx\n\
+sleep 5\n\
+curl -X POST http://127.0.0.1:3928/inferences/llamacpp/loadmodel \\\n\
+  -H "Content-Type: application/json" \\\n\
+  -d @/app/model/loadmodel.json\n\
+wait' > /start.sh && chmod +x /start.sh
+
+COPY loadmodel.json /app/model/loadmodel.json --chown=user
+
+ENV PATH="/app:${PATH}"
+
+ENV HF_HOME="/data/.huggingface"
+
+EXPOSE 80
+
+CMD ["/start.sh"]

README.md

@@ -1,11 +1,20 @@
 ---
-title: Nitro AquilaChat2 7B 16K GGUF
-emoji: 📊
-colorFrom: yellow
-colorTo: green
+title: Nitro AquilaChat2-7B-16K-GGUF
+emoji: 👀
+colorFrom: blue
+colorTo: blue
 sdk: docker
 pinned: false
 license: mit
+short_description: AquilaChat2-7B-16K-GGUF, loaded by nitro.
+app_port: 80
+models: mzwing/AquilaChat2-7B-16K-GGUF
+custom_headers:
+  cross-origin-embedder-policy: cross-origin
+  cross-origin-opener-policy: cross-origin
+  cross-origin-resource-policy: cross-origin
+preload_from_hub:
+  - mzwing/AquilaChat2-7B-16K-GGUF AquilaChat2-7B-16K.Q8_0.gguf
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+AquilaChat2-7B-16K-GGUF, loaded by nitro.

loadmodel.json

@@ -0,0 +1,4 @@
+{
+  "llama_model_path": "~/.cache/huggingface/hub/mzwing/AquilaChat2-7B-16K-GGUF/AquilaChat2-7B-16K.Q8_0.gguf",
+  "ctx_len": 16384
+}

nginx.conf

@@ -0,0 +1,41 @@
+worker_processes auto;
+
+events {
+  worker_connections 1024;
+}
+
+http {
+  include mime.types;
+  default_type application/octet-stream;
+
+  sendfile on;
+  keepalive_timeout 65;
+
+  proxy_connect_timeout 600;
+  proxy_send_timeout 600;
+  proxy_read_timeout 600;
+  send_timeout 600;
+
+  map $http_upgrade $connection_upgrade {
+    default upgrade;
+    '' close;
+  }
+
+  server {
+    listen 80;
+
+    location / {
+      proxy_pass http://127.0.0.1:3928;
+      proxy_set_header Host $host;
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection $connection_upgrade;
+    }
+
+    location ~ ^/(inferences/llamacpp/loadmodel/|inferences/llamacpp/unloadmodel/) {
+      deny all;
+      return 403;
+    }
+  }
+}