Upload PhiMoEForCausalLM

config.json

@@ -18,7 +18,7 @@
   "input_jitter_noise": 0.01,
   "intermediate_size": 2240,
   "lm_head_bias": true,
-  "max_position_embeddings": 131072,
+  "max_position_embeddings": 4096,
   "model_type": "phimoe",
   "num_attention_heads": 32,
   "num_experts_per_tok": 2,
@@ -28,148 +28,11 @@
   "original_max_position_embeddings": 4096,
   "output_router_logits": false,
   "rms_norm_eps": 1e-05,
-  "rope_scaling": {
-    "long_factor": [
-      1.0199999809265137,
-      1.0299999713897705,
-      1.0399999618530273,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.059999942779541,
-      1.059999942779541,
-      1.059999942779541,
-      1.059999942779541,
-      1.059999942779541,
-      1.059999942779541,
-      1.0999999046325684,
-      1.1799999475479126,
-      1.1799999475479126,
-      1.3700000047683716,
-      1.4899998903274536,
-      2.109999895095825,
-      2.8899998664855957,
-      3.9499998092651367,
-      4.299999713897705,
-      6.429999828338623,
-      8.09000015258789,
-      10.690000534057617,
-      12.050000190734863,
-      18.229999542236328,
-      18.84000015258789,
-      19.899999618530273,
-      21.420000076293945,
-      26.200000762939453,
-      34.28000259399414,
-      34.590003967285156,
-      38.730003356933594,
-      40.22000503540039,
-      42.54000473022461,
-      44.000003814697266,
-      47.590003967285156,
-      54.750003814697266,
-      56.19000244140625,
-      57.44000244140625,
-      57.4900016784668,
-      61.20000076293945,
-      61.540000915527344,
-      61.75,
-      61.779998779296875,
-      62.06999969482422,
-      63.11000061035156,
-      63.43000030517578,
-      63.560001373291016,
-      63.71000289916992,
-      63.92000198364258,
-      63.94000244140625,
-      63.94000244140625,
-      63.96000289916992,
-      63.980003356933594,
-      64.0300064086914,
-      64.0300064086914,
-      64.0300064086914,
-      64.04000854492188,
-      64.10000610351562,
-      64.19000244140625,
-      64.20999908447266,
-      64.75,
-      64.95999908447266
-    ],
-    "long_mscale": 1.243163121016122,
-    "original_max_position_embeddings": 4096,
-    "short_factor": [
-      1.0,
-      1.0399999618530273,
-      1.0399999618530273,
-      1.0399999618530273,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.059999942779541,
-      1.059999942779541,
-      1.0699999332427979,
-      1.0699999332427979,
-      1.0699999332427979,
-      1.0699999332427979,
-      1.1399999856948853,
-      1.159999966621399,
-      1.159999966621399,
-      1.159999966621399,
-      1.159999966621399,
-      1.1799999475479126,
-      1.1999999284744263,
-      1.3199999332427979,
-      1.3399999141693115,
-      1.3499999046325684,
-      1.3999998569488525,
-      1.4799998998641968,
-      1.4999998807907104,
-      1.589999794960022,
-      1.6499998569488525,
-      1.71999990940094,
-      1.8999998569488525,
-      1.9099998474121094,
-      1.9099998474121094,
-      1.9899998903274536,
-      1.9999998807907104,
-      1.9999998807907104,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.0999999046325684,
-      2.319999933242798,
-      2.419999837875366,
-      2.5899999141693115,
-      2.7899999618530273
-    ],
-    "short_mscale": 1.243163121016122,
-    "type": "longrope"
-  },
+  "rope_scaling": null,
   "rope_theta": 10000.0,
   "router_aux_loss_coef": 0.0,
   "router_jitter_noise": 0.01,
-  "sliding_window": 131072,
+  "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.41.2",

model-00001-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:586ab060777d7bcf7a53abe5b0cbf55b1716a0b14386ef015ae87294e74d8134
+oid sha256:a623fee51e22a18b446a2885badaf62a4dbdf49325a61ff643be853c1c3c02a1
 size 4995293984

model-00002-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e5e79658051a9b544795dab8a2c627c1400e4899a28852e8c75e9ef697120f8e
+oid sha256:83c51b75c485019b43c73e3fea48fd558f8b2f4af3349170f158c3c0853052e3
 size 4966538712

model-00003-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:484ccc0dbc07cf3a7b70a8a4c48518cfd1f2bc88860b845e024ff488ebbb7de2
+oid sha256:48e1b8ceede8d80f965076e366c421e07f48fa58ba7177ca963b26c4c4d430e8
 size 4986555656

model-00004-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5ffd5ce49775737d361803d3ea853d8dfe60f3546299e81d5d743c4d794143c7
+oid sha256:2d5197aadddd4601801ae608b8faa458ce811740b70f4a9519e82b6faaad24e9
 size 4989562032

model-00005-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ca7c3d7023cee93f1da3947fdbf40ee5472b780be7b8867988fb8fc01f13dd36
+oid sha256:9a6ac6179b129441fcf91f3766a96b06756aa0caf9bc1ded2849a39578fc3f6e
 size 4989562032

model-00006-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:575b96d6d469ea226ff25bef40082dcb48ca526c26670eefaa14df9e76ce7785
+oid sha256:821c6a3dec0728283a595299988ff6910d364ea9aa5fbb951d47f99690620143
 size 4989562040

model-00007-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f148de3a6c7f8c8bf7d453a7d18a039a740a1ce57da188226c3ccc3acfc4219b
+oid sha256:cfdd16e74d2b4fc03ed66ada75a375cba0c3d5d975c2eaf79d1abf3116a0a791
 size 1484567336