OpenPirate V2. Much better performance on a wide range of prompts.

config.json

@@ -1,7 +1,7 @@
 {
   "_name_or_path": "/models/latent-descent/base",
   "architectures": [
-    "MistralForCausalLM"
+    "SplitAttentionForCasualLM"
   ],
   "bos_token_id": 1,
   "eos_token_id": 32000,
@@ -14,6 +14,7 @@
   "num_attention_heads": 32,
   "num_hidden_layers": 32,
   "num_key_value_heads": 8,
+  "output_attentions": true,
   "rms_norm_eps": 1e-05,
   "rope_theta": 10000.0,
   "sliding_window": 4096,

generation_config.json

@@ -2,6 +2,7 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 32000,
+  "output_attentions": true,
   "transformers_version": "4.35.2",
   "use_cache": false
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1db494bb42cc27e565dc25caee298ba2ddb42c19b13a67f02daeb07a183f0d8d
+oid sha256:0494ed32f7fbe9427179b5c00796c83ca88ea881397d695cdeb70e9b0ac19087
 size 4943178624

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3b66024c1fbb405ecaf91802ed578e57abf3309988f2ba383ffef86ee4d73a91
+oid sha256:21a49b7d546ea7f45b68efc25440a2cf6402029096fe154e9bc3ae365541ffcb
 size 4999819232

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:af8a791e792058932accc55fd78292298d42c74839cb774c385e158e77f453bb
+oid sha256:5baa07560a15271448be3be97905f75b742476ac5320bd3c31d2e3fd2cdb89e4
 size 4540532640