{
"measurement": [
{
"key": "model.layers.0.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.01697666384279728,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.016824379563331604,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.008012637495994568,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.007979977875947952,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.007946329191327095,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.0049625481478869915,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.02453072927892208,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.01671225018799305,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.007964342832565308,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.00792560912668705,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.00843674037605524,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.009553074836730957,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.007916495203971863,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.0055845873430371284,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.004945029970258474,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.005946503020823002,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.004938302095979452,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.004719415679574013,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.0049372403882443905,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.004717701580375433,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.00512714171782136,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.00493657449260354,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.004871716257184744,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004716654308140278,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.0.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.017844613641500473,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.017602531239390373,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.00851193442940712,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.008455167524516582,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.008397861383855343,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.004057686775922775,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.02155294641852379,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.017445094883441925,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.008435689844191074,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.008356738835573196,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.00860097911208868,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.009192141704261303,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.008340772241353989,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.00511879613623023,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.004014347679913044,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.005068568047136068,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.003994891420006752,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.0035852924920618534,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.00399156054481864,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.0035800118930637836,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.0036689562257379293,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.003990328870713711,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.0031288396567106247,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.0035771990660578012,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.0.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.058581531047821045,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.048666201531887054,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.03854528069496155,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.032693520188331604,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.026008691638708115,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.01865428499877453,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.04778392240405083,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.0373171791434288,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.02774505876004696,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.021324293687939644,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.021746069192886353,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.025533661246299744,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.01810801587998867,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.012998726218938828,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.011404740624129772,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.012817160226404667,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.007351329084485769,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.006644937675446272,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.0065429857932031155,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.005608722101897001,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.006643940228968859,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.0060324901714921,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.003913337364792824,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004441624507308006,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.0.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.02250208705663681,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.013569669798016548,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.007884315215051174,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.009216784499585629,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.008847703225910664,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.0037343567237257957,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.014458845369517803,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.012818370014429092,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.010584532283246517,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.0065164403058588505,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.006918524391949177,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.007375629153102636,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.006371545605361462,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.004712981171905994,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.004239179193973541,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.003861779347062111,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.003146079834550619,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.003025887068361044,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.0028449026867747307,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.0026164106093347073,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.002396766794845462,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.00282901874743402,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.001953230705112219,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.002489144681021571,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.0.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.06665952503681183,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.06252993643283844,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.06120281293988228,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.05572009086608887,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.029863500967621803,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.028597885742783546,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.03325073793530464,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.03076876699924469,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.030232366174459457,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.02727220207452774,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.026022255420684814,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.01687612384557724,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.014837482944130898,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.014441785402595997,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.014351986348628998,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.008516911417245865,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.007918255403637886,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.007890370674431324,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.007459202781319618,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.0074049183167517185,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.0048074014484882355,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.005444404669106007,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.004678822588175535,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.004351944662630558,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.0.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.09868097305297852,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.09270312637090683,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.09084239602088928,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.08270438015460968,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.04420515149831772,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.04238166660070419,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.04911701753735542,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.045413583517074585,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.0447533093392849,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.0403631217777729,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.03854478895664215,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.02477056346833706,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.021557675674557686,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.021040132269263268,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.020914506167173386,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.012379040941596031,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.010962944477796555,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.010923045687377453,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.010233821347355843,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.01015141885727644,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.006602860521525145,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.0068350727669894695,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.006419521756470203,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.004813126288354397,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.0.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.05961788818240166,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.049467235803604126,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.03285419940948486,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.03006347082555294,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.024773528799414635,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.016047554090619087,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.04772253334522247,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.04221887141466141,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.02659239061176777,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.021371642127633095,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.021645184606313705,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.02159249410033226,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.019264280796051025,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.013548120856285095,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.01155187375843525,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.011848951689898968,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.00904243253171444,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.008052431978285313,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.008593273349106312,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.007415123749524355,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.008195899426937103,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.008338866755366325,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.006719348486512899,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.006858801003545523,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.1.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.014748001471161842,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.011748488992452621,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.009041253477334976,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.00793392863124609,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.006322438362985849,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.004270525183528662,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.011519035324454308,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.009150850586593151,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.0068730879575014114,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.005208902060985565,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.005541016813367605,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.0060943313874304295,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.004516575485467911,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.003170345677062869,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.002736000344157219,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.003140983171761036,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.0018080300651490688,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.0016123499954119325,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.001620638882741332,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.0013596391072496772,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.0016904593212530017,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.001513556344434619,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.0010610269382596016,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.0011040866374969482,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.1.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.01392789650708437,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.010868982411921024,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.008420752361416817,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.007372470572590828,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.005823025945574045,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.0040166922844946384,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.009472841396927834,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.008341890759766102,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.00639678630977869,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.004681949969381094,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.0047745248302817345,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.004812383092939854,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.0039923787117004395,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.00287925498560071,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.0025493872817605734,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.002425770042464137,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.0016245966544374824,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.0014854806941002607,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.0014280839823186398,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.0012259215582162142,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.0013120586518198252,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.0013198396191000938,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.0009150782134383917,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.0009664052631705999,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.1.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.077581025660038,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.065501369535923,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.06035652384161949,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.05191013962030411,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.034720901399850845,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.02967154048383236,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.04300978034734726,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.039477888494729996,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.0364641398191452,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.028061265125870705,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.026248395442962646,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.021993182599544525,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.01896747015416622,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.016767917200922966,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.016215268522500992,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.011025125160813332,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.008859780617058277,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.008661223575472832,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.0075796013697981834,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.007214587181806564,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.005823236424475908,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.0059732128866016865,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.005059153772890568,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004174049943685532,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.1.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.13642467558383942,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.08754593878984451,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.0642559826374054,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.05920121446251869,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05795128643512726,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.03683771565556526,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.0815628245472908,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.07235442101955414,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.06482551246881485,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.036881621927022934,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.03910278528928757,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.04209810867905617,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.035705775022506714,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.029379338026046753,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.027687152847647667,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.021691393107175827,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.01749911531805992,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.017049770802259445,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.014273324981331825,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.013215466402471066,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01271373126655817,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01412065140902996,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.010810486041009426,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.011761549860239029,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.1.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.10389573872089386,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.09883932024240494,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.09737597405910492,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.08929068595170975,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.046213921159505844,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.04480366408824921,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.05091848224401474,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.047049958258867264,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.04660465195775032,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.04289083927869797,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.04130749776959419,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.025854021310806274,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.0224837064743042,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.022125111892819405,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.02204217202961445,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.01298458594828844,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.011874284595251083,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.01185147650539875,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.01127883791923523,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.011228898540139198,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.0072460477240383625,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.007850134745240211,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.0071310061030089855,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.006063004489988089,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.1.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.13076373934745789,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.12474245578050613,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.12296178191900253,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.11321292072534561,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.058307796716690063,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.056616779416799545,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.06410632282495499,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.05932667478919029,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.058807626366615295,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.05422080680727959,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.0521724708378315,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.03218352049589157,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.02798188477754593,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.027554819360375404,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.027455046772956848,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.016021162271499634,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.014024411328136921,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.013996395282447338,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.013224577531218529,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.013159992173314095,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.008318940177559853,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.008233299478888512,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.008172803558409214,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.0052493964321911335,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.1.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.12166032940149307,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.10948565602302551,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.09604153782129288,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.08784765750169754,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.057911161333322525,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.04296322166919708,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.07967796921730042,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.07290518283843994,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.059460338205099106,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.05291789770126343,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.05141829326748848,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.04384872689843178,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.04140756279230118,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.02644788846373558,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.023997541517019272,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.020497899502515793,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.015577531419694424,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.014137690886855125,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.014770994894206524,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.01312511321157217,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.013149308040738106,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.0128650376573205,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.01081382017582655,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.009828373789787292,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.2.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.031797830015420914,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.029201941564679146,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.028020469471812248,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.02505553513765335,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.014372274279594421,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.013305770233273506,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.01737665757536888,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.015539905056357384,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.014648344367742538,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.012720649130642414,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.012106462381780148,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.008870439603924751,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.007459557615220547,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.006904521491378546,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.006764437071979046,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.004492495208978653,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.0036375001072883606,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.003572472371160984,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.003320419928058982,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.0032322967890650034,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.002421110635623336,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.0023823671508580446,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.002167848637327552,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.001685780705884099,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.2.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.025661932304501534,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.023511024191975594,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.022562691941857338,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.02013542130589485,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.011547335423529148,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.010712658986449242,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.013626167550683022,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.012452913448214531,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.011787685565650463,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.010190362110733986,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.009640190750360489,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.006862487643957138,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.005929870065301657,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.005529738962650299,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.00543206837028265,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.003447056980803609,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.0028868538793176413,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.002845638431608677,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.0026219787541776896,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.0025607002899050713,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.0018355331849306822,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.0018487501656636596,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.0016936786705628037,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.001273848582059145,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.2.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11696620285511017,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.10897351801395416,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10641418397426605,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09617738425731659,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05365682765841484,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.05092253163456917,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.06005556881427765,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.055501945316791534,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05450619384646416,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04810641333460808,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04556171968579292,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.030424701049923897,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.026403892785310745,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.025568248704075813,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.025372043251991272,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.015170903876423836,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.013023057952523232,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.012963171117007732,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.01189382653683424,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.011768855154514313,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007830732502043247,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007702197879552841,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007523450069129467,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004836589563637972,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.2.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.1523875892162323,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.12333770841360092,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.11286048591136932,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.0878288745880127,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.06857304275035858,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.05736812204122543,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.08397381752729416,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.07598241418600082,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.07196591049432755,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.050064221024513245,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04567384347319603,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.043390288949012756,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.037255533039569855,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.03397030383348465,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.033160146325826645,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.02229878678917885,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.019290775060653687,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.019063694402575493,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.01615222543478012,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.015643445774912834,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.013164246454834938,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01427715178579092,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.012217516079545021,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.011705156415700912,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.2.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.1482105553150177,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.14117981493473053,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.13906235992908478,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.12822215259075165,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.06757542490959167,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.06547673046588898,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.07449939101934433,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.06872875988483429,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.06819659471511841,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.06264954805374146,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.06049732118844986,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.03778022900223732,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.03276849910616875,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.03226345777511597,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.032151006162166595,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.018905531615018845,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.01685652881860733,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.016827989369630814,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.01592056080698967,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.015844831243157387,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.01022054348140955,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.010539134964346886,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.010066780261695385,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.007534579839557409,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.2.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.17776700854301453,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.1693943589925766,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.1670352965593338,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.15411067008972168,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.08122321963310242,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07870887964963913,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.08943542093038559,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.08257061243057251,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.08194931596517563,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.07531553506851196,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.07271340489387512,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.04521947726607323,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.03918732330203056,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.03859851881861687,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.03845342621207237,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.022558242082595825,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.01970290206372738,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.019664796069264412,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.018525362014770508,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.01843975856900215,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.011856491677463055,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.011623648926615715,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.011655637994408607,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.007498105987906456,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.2.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.17058540880680084,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.15451598167419434,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.14855581521987915,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.134848952293396,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.07645531743764877,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.07066633552312851,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.0889575183391571,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.08178640156984329,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.07876504957675934,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.06786482036113739,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.06504137068986893,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.04507603868842125,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.039003636687994,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.0366152748465538,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.03603672236204147,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.02259993925690651,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.01915287785232067,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.019029254093766212,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.017490826547145844,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.01712738163769245,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.012201663106679916,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.012279038317501545,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.01143889781087637,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.008572560735046864,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.3.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.04428208991885185,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.040579892694950104,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.03905409947037697,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.035125453025102615,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.020013805478811264,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.018659740686416626,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.02401878498494625,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.021520327776670456,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.020639672875404358,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.017865212634205818,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.017078407108783722,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.0123124485835433,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.01033099927008152,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.009618628770112991,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.009438613429665565,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.006232317537069321,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.0050392188131809235,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.004987342283129692,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.004620281048119068,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.004507845733314753,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.003345692064613104,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.0032573649659752846,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.0030125752091407776,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.002269246382638812,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.3.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.035717517137527466,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.0330108180642128,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.03175613656640053,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.02853766269981861,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.016265686601400375,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.015182693488895893,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.01913699507713318,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.017514411360025406,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.016582611948251724,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.014500650577247143,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.01387078408151865,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.00968004111200571,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.008356825448572636,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.007816721685230732,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.007683966774493456,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.00485198711976409,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.004108237102627754,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.004052333999425173,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.003764546010643244,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.003683765884488821,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.002591368043795228,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.0026587466709315777,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.002405153587460518,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.0018783891573548317,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.3.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.15813371539115906,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.14793835580348969,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.1446092277765274,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.13100838661193848,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.07310453057289124,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.06954086571931839,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.08225326240062714,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.07549132406711578,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.07419280707836151,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.06573486328125,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.06255479902029037,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.04170239716768265,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.0359744094312191,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.03490135818719864,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.034647829830646515,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.02081129513680935,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.017772190272808075,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.017691198736429214,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.01626143418252468,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.016099734231829643,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.010784495621919632,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.010502427816390991,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.010370563715696335,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.006593007128685713,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.3.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.12367028743028641,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1065722331404686,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.09618096798658371,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.07814522087574005,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.056345995515584946,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04776884615421295,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.08090584725141525,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.06781283766031265,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05852026492357254,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04592828080058098,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.041973210871219635,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.041292767971754074,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.033958807587623596,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.028928330168128014,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.027588145807385445,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.02208034321665764,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.017776209861040115,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.017229687422513962,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.016166331246495247,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.015422974713146687,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.013780932873487473,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.014570636674761772,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.012021937407553196,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.012557935900986195,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.3.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.16777007281780243,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.15920232236385345,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.15665274858474731,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.14379490911960602,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.07805180549621582,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.0752665251493454,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.08609108626842499,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.07957065850496292,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.07886120676994324,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.07169540226459503,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.0689154639840126,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.044062789529561996,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.03837786614894867,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.03770415112376213,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.03754674270749092,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.022090591490268707,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.020194508135318756,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.02016146294772625,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.019003896042704582,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.018913859501481056,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.012195101007819176,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.013288667425513268,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.011983620934188366,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.010171458125114441,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.3.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.20330099761486053,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.19303150475025177,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.18996503949165344,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.17444491386413574,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.09437263011932373,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.0910460501909256,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.10408295691013336,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.0961853638291359,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.09536095708608627,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.08672675490379333,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.08327201008796692,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.05293827876448631,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.045931339263916016,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.04511265456676483,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04493102431297302,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.026446960866451263,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02331710420548916,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.023262226954102516,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.021792354062199593,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02167246863245964,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.014075336046516895,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014198733493685722,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.01380026526749134,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009678437374532223,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.3.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.024547332897782326,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.02149895206093788,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.02139224298298359,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.019144121557474136,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.013921414501965046,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.009887623600661755,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.015065362676978111,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.013472085818648338,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.01432189904153347,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.012498436495661736,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.012113329023122787,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.011808639392256737,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.010216418653726578,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.005469846539199352,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.005545740481466055,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.003615187481045723,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.003416619962081313,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.0030035292729735374,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.003130461322143674,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.0026302356272935867,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.002589229727163911,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.00268166814930737,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.0018744954140856862,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.0017345327651128173,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.4.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.05728099122643471,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.05266595259308815,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.05079251900315285,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.04555576667189598,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.026136938482522964,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.024369575083255768,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.03018818236887455,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.027822548523545265,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.026660103350877762,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.023115074262022972,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.021903514862060547,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.015273717232048512,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.013255803845822811,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.012511475011706352,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.01232750341296196,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.007635389920324087,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.0064753876067698,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.006401902064681053,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.005876530427485704,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.005763320717960596,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.003995359875261784,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.004042140208184719,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.0037431949749588966,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.002709820168092847,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.4.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.05086640641093254,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.046725932508707047,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.0449954979121685,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.04033806547522545,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.023193281143903732,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.02158297598361969,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.02703419327735901,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.024792125448584557,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.023666124790906906,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.02050616592168808,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.01950111612677574,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.013688706792891026,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.011812593787908554,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.01110256090760231,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.010930309072136879,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.006841734517365694,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.005736980587244034,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.005667103920131922,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.00520318141207099,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.005093037616461515,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.0035782214254140854,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.0035772325936704874,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.003341563045978546,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.0023761976044625044,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.4.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.16609296202659607,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.15478162467479706,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.15108385682106018,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.13659843802452087,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.07697716355323792,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.07297646999359131,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.08638112992048264,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.07967988401651382,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.07821597158908844,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.06890702992677689,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.06525292992591858,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.04377976432442665,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.03797182813286781,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.03674914687871933,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.036456651985645294,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.021835604682564735,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.018699150532484055,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.018605196848511696,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.017038416117429733,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.01684982143342495,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.011254990473389626,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.011031168513000011,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.010803994722664356,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.0068803406320512295,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.4.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.17271888256072998,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.14989182353019714,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.14082032442092896,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.12035852670669556,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.08067116141319275,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.07106492668390274,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.09631902724504471,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.0879930928349495,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.08311514556407928,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.06534865498542786,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.05943844094872475,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.05004580318927765,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.04291941970586777,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.03959772363305092,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.03878125548362732,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.025427967309951782,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02172275446355343,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.021426495164632797,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.018906189128756523,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.018396399915218353,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.014443661086261272,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015268135815858841,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.013438757508993149,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.011931881308555603,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.4.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.1805514097213745,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.17079930007457733,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.1678871065378189,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1536673605442047,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.08397956937551498,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.08075405657291412,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.09257031977176666,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.0857488214969635,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.0849662721157074,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.07670547068119049,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.0732714906334877,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.04698094725608826,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.04090579226613045,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.04011544957756996,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.03993254527449608,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.023459061980247498,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.020645003765821457,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.02059720642864704,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.0191874448210001,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.01907055638730526,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.012309204787015915,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.012445722706615925,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.01204857137054205,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.008319983258843422,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.4.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.2153535932302475,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.2038724273443222,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.20046615600585938,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.183524027466774,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10016363114118576,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.09632957726716995,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11037316918373108,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10221032053232193,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.10129869729280472,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09147801250219345,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.08740895241498947,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.055947110056877136,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.048668164759874344,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.04774129390716553,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04752131178975105,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.027882788330316544,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.0242391936480999,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.024183502420783043,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02246173471212387,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02231917344033718,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.014383867383003235,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014100656844675541,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.014071588404476643,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.00873632449656725,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.4.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.21214354038238525,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.19174739718437195,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.18446683883666992,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.16481325030326843,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.09677419066429138,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.08932953327894211,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.11146336793899536,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.10271294414997101,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.09949786216020584,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.08442748337984085,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.07967159897089005,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.05668652802705765,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.04908811300992966,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.04641139134764671,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.04576992243528366,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.02839936874806881,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.024198472499847412,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.02407347410917282,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.02177801914513111,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.02136961556971073,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.015235237777233124,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.015318303368985653,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.014393865130841732,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.010585842654109001,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.5.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.0674562007188797,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.06203921139240265,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.05972308665513992,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.053530123084783554,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.03083694726228714,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.028726452961564064,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.035685304552316666,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.03297979757189751,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.03147094324231148,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.027270931750535965,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.025789404287934303,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.018070627003908157,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.01574786938726902,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.01479566004127264,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.01456593070179224,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.009042159654200077,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.007711011916399002,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.007617549505084753,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.007007702253758907,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.006863470654934645,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.0047660949639976025,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.004899340216070414,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.004450256936252117,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.003374494146555662,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.5.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.05894502252340317,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.054171472787857056,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.052077680826187134,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.04665432125329971,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.026975804939866066,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.025060970336198807,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.03162145987153053,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.029023105278611183,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.027522239834070206,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.023855077102780342,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.022668685764074326,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.01601143181324005,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.013844946399331093,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.012939582578837872,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.012717332690954208,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.008009051904082298,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.006717710290104151,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.006626029498875141,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.006100108847022057,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.005961676128208637,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.004206099081784487,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.004247805569320917,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.003903669770807028,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.0028754582162946463,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.5.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.1763375997543335,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.16417944431304932,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.1601824015378952,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.14467817544937134,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.08185044676065445,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.07753173261880875,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.092045359313488,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.08482979983091354,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.08318497240543365,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07315465807914734,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.06924226135015488,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.04669727385044098,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.04045300930738449,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.039090439677238464,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.03877197206020355,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.023304497823119164,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.019922098144888878,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.019813233986496925,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.018137505277991295,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.017928041517734528,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.012040653266012669,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.011806510388851166,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.011539632454514503,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.007401570212095976,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.5.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.17501622438430786,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.15546874701976776,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.14861012995243073,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.12490829080343246,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.08185647428035736,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.07432493567466736,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.09504301100969315,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.08679988235235214,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.08389441668987274,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.06717049330472946,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.0605769008398056,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.04907383397221565,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.04228176549077034,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04005768895149231,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.039523471146821976,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.024928005412220955,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.021898703649640083,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.021719401702284813,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.019176984205842018,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.01885000616312027,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.014102224260568619,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015119810588657856,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.013430260121822357,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.011861717328429222,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.5.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.16609011590480804,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.15672938525676727,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.15398810803890228,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.14063230156898499,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.0775030180811882,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07434995472431183,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.0856412798166275,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.07926401495933533,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.07843594253063202,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.07049022614955902,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.06729941070079803,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.04358799383044243,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.037931859493255615,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.03713399916887283,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.03694628179073334,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.0217838566750288,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.019289299845695496,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.01924092508852482,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.017897218465805054,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.017781583592295647,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.011557992547750473,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.011899665929377079,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.011294588446617126,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.008264943957328796,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.5.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.22015579044818878,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.20798249542713165,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.20430287718772888,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.18670620024204254,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10269824415445328,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.0985633060336113,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11345574259757996,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10500314086675644,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.1039678081870079,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.0934738963842392,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.08913198113441467,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.057545408606529236,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.0500381775200367,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.04900835454463959,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04876282438635826,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.028702132403850555,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.024906400591135025,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.02484174259006977,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.023000312969088554,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.022842969745397568,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.014850955456495285,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014538484625518322,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.014498659409582615,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009044058620929718,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.5.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.21345894038677216,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.19291165471076965,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.18555045127868652,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.16551589965820312,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.09757258743047714,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.0900067687034607,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.11266666650772095,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.10373701900243759,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.10034715384244919,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.08512851595878601,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08010753244161606,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.0573735237121582,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.04967370629310608,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.04689139127731323,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.04621157422661781,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.02880791202187538,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.02454410493373871,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.02441766858100891,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.022116562351584435,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.021691108122467995,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.015591393224895,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.015691407024860382,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.014698616228997707,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.011003565974533558,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.6.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.08186301589012146,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.07611765712499619,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.07381702214479446,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.06657375395298004,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.03782615065574646,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.03561867028474808,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.043120723217725754,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.039818573743104935,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.038461461663246155,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.033793702721595764,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.0319998674094677,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.021879345178604126,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.019037999212741852,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.01815960370004177,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.017948126420378685,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.010957115329802036,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.009449400007724762,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.009367168880999088,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.008649191819131374,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.008518526330590248,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.005787377245724201,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.005938382353633642,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.005489308387041092,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004101419821381569,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.6.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.0688747987151146,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.06401612609624863,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.06203761696815491,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.056020863354206085,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.03184346482157707,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.02994154952466488,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.03673607110977173,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.03369339555501938,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.03238595277070999,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.02846824936568737,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.027118001133203506,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.018635889515280724,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.01608731597661972,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.015262428671121597,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.015063817612826824,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.0093284472823143,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.007889430038630962,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.007810463663190603,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.007215285673737526,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.0070924535393714905,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.0048910947516560555,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.004898827522993088,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.0046106548979878426,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.0032794279977679253,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.6.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.19314824044704437,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1807585209608078,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.17672410607337952,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.15994301438331604,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.09000424295663834,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.08555044233798981,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.10077493637800217,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.09296105802059174,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.09135895222425461,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.08085043728351593,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.07654497772455215,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.05110635608434677,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.04432695731520653,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04299145191907883,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04267479106783867,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.025496039539575577,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02184523642063141,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.021746113896369934,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.01995226927101612,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.01974262297153473,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.013115225359797478,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.012831345200538635,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.012616008520126343,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.007932804524898529,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.6.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.17536561191082,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.149716317653656,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.13935676217079163,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.11514901369810104,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.08165270835161209,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.07092972099781036,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.09788499027490616,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.08970655500888824,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.08416131883859634,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.06339824944734573,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.05704877898097038,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.0505555234849453,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.04364539682865143,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.03998519107699394,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.039081405848264694,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.025412872433662415,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.021883340552449226,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.021540310233831406,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.018540311604738235,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.0179671049118042,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.014171162620186806,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015366354957222939,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.013029881753027439,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.011926187202334404,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.6.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.1625240594148636,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.1532370150089264,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.15039977431297302,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1373308300971985,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.07597769051790237,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07279747724533081,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.08402539044618607,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.07782621681690216,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.07692167162895203,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.06895845383405685,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.06573671847581863,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.04270012676715851,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.037180230021476746,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.036352768540382385,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.036155976355075836,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.02133001573383808,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.01870124600827694,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.018647450953722,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.01728944107890129,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.01716652140021324,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.011182788759469986,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.01128343679010868,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.010904481634497643,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.007512768264859915,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.6.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.22308506071567535,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.21054302155971527,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.20672881603240967,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.18878401815891266,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10431936383247375,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.1000388041138649,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11540082097053528,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10679644346237183,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.10564403235912323,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09476101398468018,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09039802104234695,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.05861181393265724,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05094125121831894,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.049842435866594315,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04957412928342819,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.029267776757478714,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02537979558110237,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.025305096060037613,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.023411009460687637,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.023241877555847168,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.015227214433252811,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014896470122039318,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.01485538762062788,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009381753392517567,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.6.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.2177283614873886,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.19590069353580475,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.18787802755832672,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.16769221425056458,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.09943516552448273,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.09126951545476913,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.11558470129966736,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.1063755601644516,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.10246603935956955,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.08645522594451904,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08156789094209671,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.05883955955505371,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.050938792526721954,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.047822244465351105,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.04707130789756775,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.02951131761074066,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.02510569617152214,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.02494862489402294,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.022586848586797714,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.022109640762209892,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.015974288806319237,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.01618822291493416,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.0149933360517025,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.011418033391237259,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.7.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.08034785836935043,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.07478385418653488,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.07260110974311829,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.06549286842346191,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.03713182732462883,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.03496406227350235,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.04238475486636162,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.03910502418875694,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.03772877901792526,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.03318791463971138,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.03149624541401863,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.021471375599503517,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.018673233687877655,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.017789272591471672,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.017572494223713875,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.01073844451457262,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.009186962619423866,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.009100396186113358,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.008399334736168385,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.008263207040727139,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.005623695440590382,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.0056769391521811485,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.005320101510733366,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.0037978547625243664,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.7.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.06861290335655212,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.06390445679426193,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.06187458336353302,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.05582408234477043,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.03172047808766365,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.029802709817886353,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.036531463265419006,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.033652208745479584,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.03223009407520294,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.02838413044810295,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.02701103687286377,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.018517807126045227,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.01607285439968109,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.0151955122128129,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.01498408243060112,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.009257826022803783,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.007840887643396854,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.007750756107270718,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.007173503749072552,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.007039862219244242,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.0048347036354243755,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.0048556700348854065,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.004543744958937168,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.0032122586853802204,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.7.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.1898217797279358,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1776759922504425,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.1736973375082016,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.15727828443050385,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.08850862085819244,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.08411891013383865,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.0993950366973877,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.09144212305545807,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.08980666100978851,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07946421205997467,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.07539424300193787,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.05047943815588951,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.043599873781204224,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04226969927549362,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04195089638233185,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.025180771946907043,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.021487977355718613,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.021384790539741516,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.01962202414870262,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.019411491230130196,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.012963681481778622,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.012633866630494595,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.012428310699760914,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.007835621945559978,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.7.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.1996597945690155,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.17305852472782135,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.1635490357875824,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.13872189819812775,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.09138768911361694,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.08120381087064743,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.10921651124954224,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.09956055134534836,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.09532192349433899,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.0752684697508812,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.0680159330368042,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.05623292177915573,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.04867368936538696,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04493832215666771,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04405278339982033,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.0285743810236454,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.024997549131512642,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.024745360016822815,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.022100454196333885,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02155911549925804,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.016323495656251907,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.017911700531840324,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.015205970965325832,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.014330457895994186,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.7.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.1523459404706955,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.14361000061035156,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.1409284472465515,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.12849204242229462,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.07132120430469513,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.06826495379209518,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.07897891104221344,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.07314279675483704,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.07222668081521988,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.06468478590250015,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.0616462379693985,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.040163554251194,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.034984804689884186,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.03416406735777855,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.03396708518266678,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.02007446624338627,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.01764446310698986,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.01758560910820961,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.016310635954141617,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.01618952304124832,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.010553066618740559,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.010751024819910526,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.010284971445798874,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.007283943239599466,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.7.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.21958614885807037,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.2071847766637802,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.20341837406158447,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.18571139872074127,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10301312804222107,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.09869552403688431,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.1140298917889595,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10552441328763962,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.1043139174580574,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.0935157909989357,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.08911700546741486,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.05802620202302933,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.050457898527383804,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.049319930374622345,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.049057699739933014,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.02897696942090988,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.025440189987421036,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.025364181026816368,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.0235231202095747,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.023358775302767754,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.01522864680737257,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.01544264703989029,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.014847972430288792,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.010404673404991627,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.7.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.21072246134281158,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.1894737035036087,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.18166254460811615,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.16178232431411743,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.09631643444299698,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.08829469978809357,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.11290658265352249,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.10299097001552582,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.09921034425497055,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.08349243551492691,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.07879514992237091,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.05726238712668419,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.049527496099472046,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.046535562723875046,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.04580829292535782,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.028945384547114372,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.024814432486891747,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.024663900956511497,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.02237817272543907,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.02192234992980957,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.016065262258052826,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.01647845283150673,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.01512020081281662,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.01217486709356308,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.8.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.0905386283993721,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.08442479372024536,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.08197401463985443,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.07400809228420258,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.04186328127980232,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.039527878165245056,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.047711119055747986,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.04403029382228851,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.042523521929979324,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.037500541657209396,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.03560240939259529,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.024182919412851334,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02101605199277401,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.020054664462804794,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.019823500886559486,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.012095448561012745,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.010358708910644054,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.010265687480568886,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.009481835179030895,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.009338252246379852,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.006346430163830519,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.006394651718437672,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.0060207899659872055,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004278238862752914,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.8.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.07536838203668594,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.07031324505805969,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.06810592114925385,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.06148509308695793,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.03486743941903114,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.032823607325553894,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.04004587233066559,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.03693149983882904,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.03541221097111702,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.03123929724097252,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.029723722487688065,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.020292583853006363,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.01763204298913479,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.01670040749013424,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.01648145355284214,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.010151444002985954,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.008622257970273495,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.008523937314748764,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.007895507849752903,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.0077539971098303795,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.005316888447850943,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.005336225964128971,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.005014798603951931,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.003541011828929186,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.8.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.20386239886283875,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.190961092710495,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.18672502040863037,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.16917136311531067,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.09507286548614502,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.09044045209884644,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.10696837306022644,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.09818030148744583,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.09644611179828644,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.08540654182434082,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.08121728897094727,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.054339081048965454,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.04681798443198204,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04539540037512779,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.045056670904159546,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.027101140469312668,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02308260090649128,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02296745404601097,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.0210886150598526,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.020867295563220978,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.013964726589620113,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.013563130050897598,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.013374097645282745,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.008398130536079407,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.8.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.19204667210578918,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1699705719947815,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.16237515211105347,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.13812559843063354,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.08937272429466248,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.08091378957033157,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.1032981351017952,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.09487491101026535,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.09190496802330017,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07371944934129715,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.06650415062904358,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.053459975868463516,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.04627373069524765,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04382512718439102,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04323640838265419,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.027073094621300697,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02403770200908184,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.023851869627833366,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.021140217781066895,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02078840509057045,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.015399452298879623,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.016697628423571587,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.014683022163808346,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.0131946736946702,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.8.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.1608821153640747,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.15165984630584717,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.14878980815410614,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.13569243252277374,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.07552041858434677,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07224614918231964,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.08367972820997238,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.07745105773210526,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.07644795626401901,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.06848523020744324,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.0652657225728035,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.042705047875642776,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.03716372698545456,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.03627277910709381,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.03606535121798515,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.02135094441473484,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.018913621082901955,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.01884744130074978,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.017523126676678658,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.017392000183463097,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.011374675668776035,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.011792807839810848,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.011086314916610718,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.008308402262628078,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.8.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.22119995951652527,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.20870456099510193,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.20484256744384766,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.18695683777332306,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10361732542514801,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.09921856969594955,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11488566547632217,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10618588328361511,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.1049199178814888,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09402593225240707,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.0896017998456955,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.05834835395216942,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.050673339515924454,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.04950827360153198,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04923805221915245,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.02913958579301834,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02523214928805828,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.025152437388896942,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.023263130336999893,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.023083830252289772,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.015165315009653568,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.0148592134937644,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.014763034880161285,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009388921782374382,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.8.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.22271007299423218,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.19982574880123138,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.19149979948997498,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.17037862539291382,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.1019226610660553,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.09329868108034134,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.11874142289161682,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.10900844633579254,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.10505560040473938,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.08814021199941635,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08295983076095581,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.060401588678359985,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.052390892058610916,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.0492057166993618,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.04844493418931961,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.030388137325644493,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.02610926702618599,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.02595900371670723,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.02346312627196312,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.022977424785494804,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.016582544893026352,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.017186550423502922,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.01556538138538599,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.012525621801614761,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.9.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.09845387935638428,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.09184538573026657,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.08922264724969864,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.08072017878293991,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.045667145401239395,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04312656447291374,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05202378332614899,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.04798451438546181,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.046374138444662094,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04096756502985954,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.038930244743824005,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.026480773463845253,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.023033535107970238,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.021990839391946793,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.021739086136221886,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.013275960460305214,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.011563730426132679,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.01146914716809988,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.010639443062245846,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.010488487780094147,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007083322387188673,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007417903281748295,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.006737976800650358,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005311781540513039,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.9.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.08476117998361588,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.07911670953035355,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.07674600183963776,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.06942322850227356,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.039309192448854446,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.037056952714920044,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.045056603848934174,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.04154117405414581,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.03991620987653732,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.03527256101369858,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.033597614616155624,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.02287658117711544,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.019841518253087997,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.018850376829504967,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.018609048798680305,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.011438429355621338,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.009744678623974323,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.009642942808568478,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.008933092467486858,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.008785158395767212,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.006000467576086521,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.006041265092790127,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.005665494594722986,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004048591013997793,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.9.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.20188309252262115,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.18909981846809387,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.18488025665283203,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.16771607100963593,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.0941985547542572,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.08958685398101807,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.106107197701931,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.09737023711204529,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.09555796533823013,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.08471689373254776,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.08052194118499756,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.05388044938445091,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.04646044969558716,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.045028313994407654,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04466996341943741,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.026885252445936203,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02292756922543049,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02281452715396881,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.020976586267352104,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02075263299047947,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01389752235263586,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.013553394004702568,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.013318858109414577,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.0085020512342453,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.9.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.20123377442359924,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1752019226551056,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.1644483506679535,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.13546934723854065,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.09248624742031097,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.08233442902565002,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11261624842882156,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10272619873285294,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.09619056433439255,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07500435411930084,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.06757707893848419,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.05793517827987671,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05037793144583702,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.0457594208419323,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04463893547654152,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.02944871224462986,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02580363303422928,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.025414984673261642,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.022754235193133354,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.0220626313239336,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.016934465616941452,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01896543614566326,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.015535200946033001,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.015391943044960499,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.9.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.165664404630661,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.15617266297340393,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.15319772064685822,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.13974161446094513,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.07775119692087173,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07438155263662338,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.0862317904829979,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.07983114570379257,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.0787426233291626,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.07046826928853989,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.06714973598718643,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.04395756497979164,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.03826827555894852,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.037331391125917435,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.03710591048002243,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.021969525143504143,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.019427821040153503,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.01936260797083378,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.017982903867959976,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.01784714311361313,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.011648500338196754,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.012065884657204151,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.011343706399202347,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.008434731513261795,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.9.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.22523090243339539,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.21242569386959076,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.20847004652023315,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1902206689119339,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10552956163883209,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10102353990077972,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11703088879585266,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10820505023002625,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.10688003897666931,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09569434821605682,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09113765507936478,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.059425078332424164,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.051639992743730545,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.05043517053127289,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05014393851161003,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.029690243303775787,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.025706904008984566,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.025623474270105362,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02367744967341423,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.023497460409998894,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.015449518337845802,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.01513752806931734,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.015030693262815475,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.00956702884286642,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.9.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.23129518330097198,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.2078687846660614,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.19929727911949158,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.17719252407550812,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.10609383136034012,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.09726838022470474,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.12355038523674011,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11344242841005325,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.1093183234333992,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09182684868574142,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08637087047100067,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.0630621612071991,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.054609328508377075,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.051273033022880554,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.05047620087862015,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.031741801649332047,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.0273037850856781,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.027146115899086,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.024584408849477768,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.024082813411951065,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.017450444400310516,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.018100686371326447,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.016393447294831276,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.013348204083740711,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.10.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.10491836071014404,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.09792019426822662,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.09517035633325577,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.08598654717206955,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.0486338809132576,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04596574977040291,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.055244408547878265,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05104833096265793,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.049397923052310944,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04359063133597374,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.0413355827331543,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.02805149555206299,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.024419991299510002,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.023338042199611664,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.023079125210642815,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.014036566950380802,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.0121234692633152,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.01202061865478754,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.011120832525193691,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.010958073660731316,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007389173377305269,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007574733346700668,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007025556638836861,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005185587797313929,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.10.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.08663144707679749,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.0808810293674469,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.07834551483392715,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.07079298049211502,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.0401076041162014,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.037780169397592545,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.04599565640091896,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.04247593507170677,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.04072241857647896,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.03597336634993553,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.034182142466306686,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.023295823484659195,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02027103863656521,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.019220907241106033,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.018967293202877045,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.01165719609707594,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.009933258406817913,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.009825542569160461,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.009102551266551018,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.008943229913711548,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.0061145662330091,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.006163831800222397,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.0057736774906516075,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004118406679481268,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.10.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.20439743995666504,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1914300173521042,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.1868240088224411,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.16920378804206848,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.09533128142356873,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.09049317240715027,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.10723628848791122,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.09889993816614151,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.09669911861419678,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.08562135696411133,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.08115634322166443,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.05447028577327728,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.04718348756432533,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04553218185901642,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04513484239578247,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.02716362290084362,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02321287989616394,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.023068198934197426,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.02122345007956028,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.0209652129560709,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.014034408144652843,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.013784538954496384,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.013431234285235405,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.00864663440734148,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.10.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.21774964034557343,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1928023099899292,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.18354356288909912,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.15679194033145905,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10083477199077606,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.09118327498435974,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11861319094896317,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10894033312797546,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10433042794466019,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.08430022746324539,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.07656096667051315,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.0613023079931736,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.053230009973049164,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04954170063138008,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04865150898694992,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03113100491464138,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.027427246794104576,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02716631069779396,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.024421213194727898,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.023889994248747826,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01776622235774994,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01945824921131134,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.016680505126714706,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.015514731407165527,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.10.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.16124160587787628,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.15186291933059692,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.14883436262607574,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.13558082282543182,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.07571621239185333,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07239073514938354,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.08407186716794968,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.07778812199831009,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.07669733464717865,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.06855633109807968,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.06526140123605728,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.042907021939754486,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.03736378252506256,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.03643021732568741,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.03620687872171402,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.021481025964021683,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.019092826172709465,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.019030071794986725,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.01768426224589348,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.017547164112329483,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.011508138850331306,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.012046168558299541,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.011208770796656609,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.00863878708332777,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.10.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.22328035533428192,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.21033617854118347,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.20637190341949463,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.187949538230896,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10451588779687881,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.09999148547649384,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11593224853277206,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10722099244594574,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.10586890578269958,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09462762624025345,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09000644087791443,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.05885551869869232,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05115756765007973,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.04994337260723114,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04965578392148018,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.029384998604655266,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.025447215884923935,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.025363245978951454,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.023409975692629814,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.023226112127304077,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.015270784497261047,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014973520301282406,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.014858514070510864,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009433685801923275,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.10.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.23422524333000183,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.21077574789524078,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.20249627530574799,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.1796295940876007,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.10743521898984909,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.09874666482210159,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.12393444776535034,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.1142609491944313,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.11053189635276794,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09294167160987854,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08713389188051224,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06331232935190201,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05495547875761986,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.05186182260513306,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.051111117005348206,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03187687322497368,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.027513809502124786,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.027372997254133224,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.0247348640114069,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.02426155097782612,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.017480704933404922,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.018068138509988785,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.01653914526104927,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.013210101053118706,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.11.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.10379376262426376,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.0968913584947586,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.0941426157951355,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.08511965721845627,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.04815684258937836,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04547487571835518,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.0547710619866848,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.050617676228284836,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.048885002732276917,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04315732792019844,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04092932492494583,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.02779443934559822,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.024176018312573433,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.023069310933351517,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.02281242422759533,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.013897554948925972,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.011923879384994507,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.011816405691206455,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.01092109177261591,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.010757150128483772,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007287177722901106,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007369400467723608,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.006910740863531828,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004939105361700058,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.11.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.08720480650663376,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.08144927769899368,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.07891333848237991,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.07137184590101242,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.04046136140823364,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.038112860172986984,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.04635872691869736,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.042865097522735596,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.04107111692428589,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.036286670714616776,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.03448193520307541,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.023494496941566467,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02046317793428898,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.019382784143090248,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.019118215888738632,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.011746572330594063,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.00998806394636631,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.00987818744033575,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.009149719029664993,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.008984451182186604,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.006147605367004871,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.006162232719361782,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.005801877472549677,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004063944332301617,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.11.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.20550385117530823,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.19225072860717773,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.187602236866951,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.16984395682811737,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.09581518173217773,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.09093029797077179,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.10809551924467087,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.09953910857439041,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.0972338393330574,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.08599410206079483,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.08148843050003052,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.054821182042360306,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.04746910557150841,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04578600451350212,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04538082331418991,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.02735554240643978,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.023348163813352585,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.023198211565613747,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.021332116797566414,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.021068809553980827,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.014133710414171219,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01387714222073555,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.013511131517589092,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.008724251762032509,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.11.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.21231059730052948,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.18570177257061005,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.17393609881401062,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.15247441828250885,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.09772953391075134,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.08643467724323273,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11965417861938477,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10953214764595032,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10191398113965988,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.08223003149032593,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.07663004845380783,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.061858341097831726,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05332290008664131,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04785003513097763,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.046507831662893295,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.031024549156427383,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.026189422234892845,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02570987492799759,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.023320863023400307,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02249465137720108,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.017083443701267242,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.018542852252721786,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01538562960922718,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.014261312782764435,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.11.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.16221767663955688,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.15267503261566162,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.1495528519153595,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1361556351184845,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.07615328580141068,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07275855541229248,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.0847189873456955,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.07837782800197601,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.07716305553913116,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.0689157247543335,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.06561484187841415,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.04327220097184181,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.03765837103128433,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.036667488515377045,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.036437682807445526,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.021651828661561012,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.019249852746725082,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.019179802387952805,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.017826491966843605,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.017684275284409523,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.011619579046964645,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.012203419581055641,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.011295713484287262,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.008792457170784473,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.11.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.22177229821681976,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.20875878632068634,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.20466624200344086,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.18635426461696625,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10383040457963943,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.09925220161676407,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11532934755086899,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10659419745206833,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.10518260300159454,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09392086416482925,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.08926017582416534,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.058524832129478455,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.0508752167224884,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.0496232807636261,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04932919889688492,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.029214506968855858,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02530360221862793,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.025219742208719254,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02327057160437107,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02307812124490738,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.015144039876759052,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014933713711798191,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.014706616289913654,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009462212212383747,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.11.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.2385084331035614,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.21435657143592834,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.20562584698200226,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.1818472146987915,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.1095990389585495,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.10044153034687042,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.12704697251319885,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11682964116334915,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.11280565708875656,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09442313015460968,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08830146491527557,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06504786759614944,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05619305372238159,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.05285013094544411,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.05204835161566734,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03278156369924545,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.027943387627601624,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.027784325182437897,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.025011485442519188,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.024505741894245148,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.018092108890414238,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.018243130296468735,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.01706903614103794,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.013186884112656116,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.12.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11033271253108978,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.10300339013338089,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10015387088060379,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09043625742197037,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05119280144572258,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.048385780304670334,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.058133263140916824,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05370267108082771,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.051980309188365936,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04588335379958153,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04347483441233635,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.029487494379281998,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.025639785453677177,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02451482228934765,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.02424434944987297,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.014741124585270882,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.012615070678293705,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.012505694292485714,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.011539662256836891,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.011367659084498882,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007690621539950371,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.0077109551057219505,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007307753898203373,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005065398756414652,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.12.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.0921538770198822,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.08612216264009476,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.08349303901195526,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.07544387131929398,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.042788274586200714,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04031965509057045,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.048955079168081284,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.04529932886362076,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.0434107705950737,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.038375698029994965,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.03646600991487503,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.024807313457131386,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02162552811205387,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02048955298960209,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.020214878022670746,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.012392964214086533,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.010540622286498547,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.01042248122394085,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.00965425930917263,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.009480487555265427,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.006469314452260733,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.006482116878032684,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.006112335249781609,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004245966672897339,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.12.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.21427838504314423,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.20048612356185913,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.195584237575531,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.17683137953281403,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.09997007250785828,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.09480606019496918,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11287885904312134,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10399629175662994,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10144077986478806,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.08970092982053757,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.08499222993850708,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.05733881890773773,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.04968533292412758,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.047838397324085236,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04738575965166092,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.02865908108651638,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02446950413286686,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.024298662319779396,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.02237016148865223,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02207588590681553,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.014912888407707214,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.014675315469503403,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.014240266755223274,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.009383606724441051,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.12.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.22188489139080048,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.19909003376960754,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.19056785106658936,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.16520792245864868,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10354194790124893,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.09445558488368988,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12121476233005524,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11074981093406677,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.1064358726143837,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.08754296600818634,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.08160509914159775,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06220048666000366,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05377112701535225,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.0504818856716156,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.049704648554325104,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03137339651584625,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.027396200224757195,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.027141675353050232,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.024425501003861427,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.023938441649079323,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.017408397048711777,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01878846250474453,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.016398005187511444,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.014484809711575508,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.12.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.16633374989032745,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.15652640163898468,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.1533302366733551,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.13957206904888153,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.07819398492574692,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07468518614768982,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.08703280985355377,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.08043898642063141,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.07920842617750168,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.07070204615592957,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.0673021450638771,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.04445761814713478,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.03869514912366867,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.037676721811294556,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.0374390184879303,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.022265618667006493,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.019831256940960884,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.01975705288350582,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.018364030867815018,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.018213095143437386,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.011988554149866104,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.01262939814478159,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.011656233109533787,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009172589518129826,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.12.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.22607547044754028,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.2127697467803955,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.2086338996887207,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1898968368768692,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10597115010023117,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.1012851819396019,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11776501685380936,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.1088334321975708,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.1073499396443367,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09582402557134628,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09106699377298355,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.05980094149708748,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05193747952580452,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.05064015835523605,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05032932013273239,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.029865382239222527,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.025792265310883522,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.025702765211462975,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.023691555485129356,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.023502720519900322,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.01548748929053545,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.015166196972131729,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.01503455825150013,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009528110735118389,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.12.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.24421426653862,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.21910615265369415,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.21034733951091766,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.18595753610134125,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.11210209876298904,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.10282256454229355,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.12950414419174194,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11918658763170242,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.11549121886491776,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09647531062364578,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.0903124064207077,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06634162366390228,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05732782557606697,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.054116200655698776,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.053339071571826935,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.033470269292593,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.028679054230451584,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.028559938073158264,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.025678010657429695,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.02519751898944378,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.01852390356361866,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.018799658864736557,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.01753339357674122,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.01372312381863594,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.13.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11743899434804916,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.10967478156089783,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10662920027971268,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09632320702075958,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05455780029296875,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.0515572652220726,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.061895012855529785,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.0572025366127491,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05538785085082054,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04890909790992737,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04637186974287033,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.03142520785331726,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.027380790561437607,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02618521824479103,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.0258988868445158,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.01572595164179802,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.013588341884315014,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.013474550098180771,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.012461330741643906,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.012279932387173176,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.00827834103256464,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008468194864690304,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007876281626522541,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005788436625152826,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.13.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.09591919183731079,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.08961807191371918,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.0866599828004837,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.07828251272439957,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.044472839683294296,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.041830480098724365,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.0511842742562294,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.04736834764480591,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.04515501856803894,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.03990422189235687,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.037909623235464096,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.02593437395989895,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.022595679387450218,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.021308904513716698,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.020992068573832512,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.012966787442564964,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.010982716456055641,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.01084606908261776,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.010063065215945244,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.009867455810308456,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.0067840078845620155,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.006797204725444317,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.006378185469657183,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004475926514714956,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.13.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.22293131053447723,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.20861846208572388,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.2036343663930893,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.1842460036277771,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10408566892147064,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.09877827018499374,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11731401830911636,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10805930197238922,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10560372471809387,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.09343364089727402,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.08857454359531403,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.05961276590824127,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05162172019481659,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.049787312746047974,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04934224858880043,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.029782872647047043,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.025457968935370445,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.025293860584497452,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.023283518850803375,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.022998910397291183,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01546087022870779,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015237417072057724,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01477893441915512,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.009715123102068901,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.13.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.2262570858001709,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.20245343446731567,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.19450698792934418,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.16584160923957825,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10578101128339767,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.0967356264591217,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12253876775503159,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11169890314340591,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10859648883342743,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.08873912692070007,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.08061680197715759,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06342668831348419,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.054565273225307465,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05189591646194458,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.051253292709589005,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03219657391309738,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02841014787554741,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02822825498878956,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.025232069194316864,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.024850545451045036,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.018389232456684113,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.019634997472167015,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01762985996901989,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.015471870079636574,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.13.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.16360615193843842,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.15390042960643768,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.15071919560432434,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.13721364736557007,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.07695260643959045,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07348214089870453,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.08560193330049515,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.07920024544000626,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.07799673080444336,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.06956002116203308,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.06616641581058502,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.043724026530981064,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.03808826953172684,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.037080228328704834,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.03685009106993675,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.021894564852118492,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.01948544941842556,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.01941835507750511,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.018028797581791878,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.01788913831114769,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.011759405955672264,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.012368706054985523,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.011436819098889828,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.00893473718315363,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.13.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.22838184237480164,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.214967742562294,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.21073018014431,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1917092353105545,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10707417875528336,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.1023344099521637,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11886459589004517,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10991790145635605,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.10851304978132248,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09674306958913803,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09191609174013138,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06039046123623848,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05245141685009003,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.05116597190499306,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05086154118180275,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.03014940768480301,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.026064056903123856,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.025978079065680504,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.023932920768857002,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02374061569571495,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.015656931325793266,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.015330624766647816,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.015218615531921387,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009633241221308708,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.13.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.23895588517189026,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.2145833522081375,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.2057669758796692,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.18252991139888763,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.10959552228450775,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.10043502599000931,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.12758108973503113,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11701273918151855,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.11293460428714752,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09460968524217606,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08871600031852722,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06483981013298035,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.056154556572437286,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.0528169721364975,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.05200628191232681,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03255530446767807,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.02786102145910263,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.027703499421477318,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.02496415562927723,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.024450115859508514,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.0176021046936512,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.018129274249076843,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.016530994325876236,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.01300189271569252,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.14.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11965226382017136,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.11192787438631058,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10891532897949219,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09849514812231064,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05561342090368271,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.05263020843267441,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.0629914402961731,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05823345109820366,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05644294247031212,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04993947595357895,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.047384265810251236,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.03198017552495003,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.027817480266094208,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.026626454666256905,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.026342598721385002,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.01598099246621132,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.013715628534555435,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.013599684461951256,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.012573545798659325,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.012388662435114384,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.008334328420460224,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008388223126530647,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.00792817585170269,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005538761615753174,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.14.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.09689705818891525,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.09066899865865707,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.08780518919229507,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.07945173233747482,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.0450022928416729,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04240747541189194,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.051627494394779205,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.047815848141908646,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.04566315934062004,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.040450505912303925,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.03843700513243675,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.026121651753783226,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.022798271849751472,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.021547457203269005,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.021237749606370926,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.013050795532763004,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.01107297744601965,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.010938380844891071,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.010151717811822891,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.009960275143384933,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.006795608438551426,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.006796529050916433,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.006404671352356672,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004407551139593124,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.14.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.22771556675434113,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.21358822286128998,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.20828846096992493,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.1886790245771408,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10654383897781372,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.1010628268122673,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12046756595373154,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.1109384298324585,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10804801434278488,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.09587638080120087,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.09099797159433365,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06114896014332771,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05305202305316925,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.0509517677128315,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.05045138671994209,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.030560515820980072,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.026017311960458755,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.025814155116677284,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.023815587162971497,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02349798195064068,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.015853026881814003,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015533561818301678,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.015114176087081432,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.009796706028282642,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.14.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.2272055298089981,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.2081519365310669,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.20173214375972748,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.17813214659690857,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10611847043037415,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.0989314541220665,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12091022729873657,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11054939031600952,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10819936543703079,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.09165169298648834,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.08536326885223389,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06253594160079956,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.053935837000608444,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05192619189620018,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.05146552249789238,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03192150965332985,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02839275635778904,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.028255676850676537,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.025710202753543854,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.025419043377041817,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.018508849665522575,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.019485007971525192,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.017934266477823257,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.015399228781461716,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.14.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.16927854716777802,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.15927588939666748,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.15599340200424194,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.14191144704818726,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.07957029342651367,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.0759943276643753,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.08851854503154755,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.08190636336803436,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.08064565062522888,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.07191866636276245,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.06839064508676529,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.04516609385609627,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.03932596370577812,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.03827756270766258,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.03802931681275368,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.02258320152759552,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.020014498382806778,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.019939443096518517,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.018505023792386055,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.018353451043367386,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.012043120339512825,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.01257265079766512,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.011702234856784344,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.0089236656203866,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.14.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.2297023981809616,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.21622709929943085,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.21204069256782532,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.19297532737255096,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10773895680904388,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10297902673482895,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11960016191005707,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.11061467975378036,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.10916091501712799,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09740686416625977,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.0925772562623024,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.060720521956682205,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05281483009457588,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.05150555819272995,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05119664594531059,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.030307741835713387,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02624719962477684,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.0261529590934515,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02411206252872944,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.023914724588394165,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.015689801424741745,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.015450137667357922,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.015238132327795029,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009729037992656231,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.14.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.24227222800254822,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.21683615446090698,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.20769010484218597,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.1838131695985794,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.11087065190076828,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.10136738419532776,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.1288461983203888,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11851020157337189,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.11441143602132797,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.0953463762998581,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08935990929603577,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.0656573474407196,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.056784793734550476,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.05330682173371315,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.052453428506851196,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03301293030381203,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.02789393812417984,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.027747241780161858,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.024862270802259445,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.024322697892785072,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.01787765510380268,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.017882157117128372,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.016783107072114944,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.012493052519857883,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.15.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11468923836946487,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.10734076797962189,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10446109622716904,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09458570182323456,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05338995158672333,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.05053457245230675,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.060546401888132095,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.055925413966178894,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05414685979485512,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04799211397767067,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.045596055686473846,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.030745480209589005,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.026724161580204964,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.025563577190041542,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.02528679370880127,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.015369710512459278,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.013147230260074139,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.013033784925937653,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.012055291794240475,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.011880271136760712,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.008020236156880856,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008020183071494102,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.0076346322894096375,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005257484968751669,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.15.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.09593964368104935,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.08981343358755112,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.08702010661363602,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.07875799387693405,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.0445941723883152,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04204832762479782,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05107513442635536,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.047359757125377655,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.04524524509906769,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04008897766470909,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.038106177002191544,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.025869062170386314,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.022587331011891365,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.021361861377954483,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.021061347797513008,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.012929846532642841,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.010993491858243942,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.010858934372663498,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.010086898691952229,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.00989674311131239,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.006750692147761583,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.006767754442989826,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.006368337199091911,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004425547085702419,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.15.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.2329186499118805,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.21856555342674255,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.21345442533493042,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.193559467792511,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10915626585483551,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.10371686518192291,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.1231101006269455,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11329212784767151,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.11064627766609192,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.09823381155729294,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.0932525098323822,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06255844235420227,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.054172780364751816,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.052206918597221375,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.05174053832888603,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03125380352139473,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02665974199771881,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.026478223502635956,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.024428419768810272,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.024122484028339386,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.016231996938586235,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01590425707399845,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01550959050655365,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.010067364200949669,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.15.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.2184024453163147,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.19767099618911743,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.18833209574222565,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.16740316152572632,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10141352564096451,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.09260617196559906,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12190502136945724,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11158781498670578,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10475526750087738,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.08874402940273285,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.08275523781776428,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06282174587249756,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05396600812673569,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04926295578479767,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04810955747961998,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.031602922827005386,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.026317739859223366,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02587730437517166,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.023871704936027527,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.023162957280874252,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.017224207520484924,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.017771080136299133,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.015710409730672836,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.013021173886954784,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.15.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.16903391480445862,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.15904758870601654,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.15581847727298737,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.14172708988189697,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.07953066378831863,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07596980780363083,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.08836633712053299,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.08181195706129074,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.08060096204280853,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.07189030200242996,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.06840910762548447,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.04508907347917557,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.03932303935289383,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.03829014673829079,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.03805308789014816,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.022580841556191444,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.020081134513020515,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.02000681683421135,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.018573710694909096,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.01842956431210041,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.012077190913259983,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.01269163005053997,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.011744190938770771,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.00911139976233244,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.15.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.2326955795288086,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.21908119320869446,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.21485744416713715,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.19563028216362,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10922835767269135,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10443494468927383,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.12126719206571579,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.11215604096651077,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.11068396270275116,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09876646846532822,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09387072175741196,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06158857420086861,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05354856699705124,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.05223521590232849,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.0519246906042099,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.0307456161826849,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.026631567627191544,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.026536161080002785,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.024473121389746666,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.024273855611681938,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.01595092937350273,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.015693655237555504,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.015504548326134682,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009917098097503185,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.15.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.24013851583003998,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.21481341123580933,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.20542602241039276,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.1819719523191452,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.11001455783843994,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.10033208876848221,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.12826211750507355,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11803960055112839,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.11356573551893234,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09457001835107803,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.0887165442109108,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06546247750520706,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05661556124687195,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.05296703428030014,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.05208444222807884,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03295135498046875,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.02782749943435192,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.027638429775834084,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.02481979690492153,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.024243909865617752,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.01790003851056099,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.01799772121012211,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.016740955412387848,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.01268034428358078,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.16.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.12405919283628464,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.11575359106063843,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.11215916275978088,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.1012255847454071,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05790690705180168,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.05452345311641693,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.06633847206830978,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.061187177896499634,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.058839209377765656,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.051784880459308624,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04915940761566162,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.03371212258934975,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.029274581000208855,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.027795590460300446,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.027436107397079468,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.016862323507666588,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.01440295297652483,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.014260930940508842,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.013181318528950214,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.012955741956830025,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.008859281428158283,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008983560837805271,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.008366107009351254,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.006075158715248108,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.16.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.1071242243051529,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.09995895624160767,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.09660378098487854,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.0871913880109787,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.049982745200395584,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04691513255238533,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05775890499353409,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05333934724330902,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05078383907675743,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04473520442843437,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.042527083307504654,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.029324006289243698,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.025497809052467346,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.023997649550437927,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.023633500561118126,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.01466613169759512,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.012410716153681278,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.012249753810465336,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.01135231088846922,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.011125000193715096,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007707131095230579,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007749568670988083,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007235053461045027,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005169446114450693,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.16.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.24302881956100464,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.22748588025569916,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.22225238382816315,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.20115730166435242,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.1139506995677948,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.10822607576847076,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12874749302864075,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.1178220734000206,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.1156742051243782,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.10216657817363739,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.09709799289703369,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06544551253318787,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.056230586022138596,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.054457906633615494,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.05403667315840721,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03267734497785568,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02770739048719406,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.027571646496653557,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.025278499349951744,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.025007551535964012,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.016841208562254906,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01632748357951641,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.016076117753982544,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.010149044916033745,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.16.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11939377337694168,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.112362802028656,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.11016415804624557,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.1008981391787529,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05594978481531143,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.05350729823112488,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.06273321062326431,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05761172994971275,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.0567452609539032,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.0508812740445137,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04895852133631706,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.03254956379532814,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.028232218697667122,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02748730406165123,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.027316153049468994,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.016690608114004135,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.015183370560407639,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.015128228813409805,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.014227936044335365,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.01412828080356121,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.009739571250975132,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.010528921149671078,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.009525960311293602,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.0084917563945055,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.16.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.17125099897384644,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.16114631295204163,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.15789051353931427,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.14358928799629211,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.08071582019329071,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07704587280750275,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.08995658159255981,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.08300620317459106,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.08178399503231049,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.07289212197065353,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.06944217532873154,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.046013351529836655,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.03988226503133774,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.03884577378630638,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.03860682621598244,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.02303159236907959,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.020344620570540428,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.020271744579076767,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.018798479810357094,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.01864817924797535,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.012406054884195328,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.012816150672733784,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.01206893939524889,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009155713953077793,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.16.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.24088510870933533,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.22673918306827545,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.22229380905628204,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.20223960280418396,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.11324963718652725,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10821309685707092,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.12612850964069366,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.11632233113050461,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.11477210372686386,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.10228034108877182,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09736334532499313,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06421884894371033,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.055574916303157806,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.054197363555431366,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05387003719806671,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.03206990286707878,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02771240845322609,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.02761818841099739,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.025460420176386833,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.0252538975328207,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.01683780923485756,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.016469448804855347,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.016367482021450996,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.010588976554572582,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.16.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.23312976956367493,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.21314507722854614,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.20599323511123657,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.18416595458984375,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.1077641099691391,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.10038073360919952,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.12404870986938477,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11378994584083557,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.11040155589580536,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09485771507024765,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08934637159109116,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06318983435630798,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05467207357287407,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.051939111202955246,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.05128505080938339,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03174315765500069,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.027381531894207,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.027244655415415764,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.024875035509467125,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.024462765082716942,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.017286021262407303,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.01768491230905056,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.016406530514359474,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.012710405513644218,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.17.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11156141012907028,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.10413648933172226,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10099941492080688,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09130469709634781,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.052129317075014114,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.049138449132442474,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05970410630106926,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05507073923945427,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05293279141187668,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04673133045434952,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04443148523569107,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.030435282737016678,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.026480956003069878,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.025138361379504204,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.024811163544654846,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.01525524165481329,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.013232587836682796,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.013102516531944275,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.012177586555480957,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.011978947557508945,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.00815505813807249,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008538208901882172,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007707811892032623,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.006120741367340088,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.17.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.1002098023891449,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.09364917129278183,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.09060017019510269,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.08198145776987076,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.04678471386432648,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.043990716338157654,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05395432561635971,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.04980158433318138,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.04750589281320572,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04197658970952034,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.03995102271437645,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.02739989012479782,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02380281314253807,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.022461414337158203,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.02213200367987156,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.013713432475924492,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.01162370853126049,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.011479778215289116,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.010657375678420067,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.01045678649097681,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007215258199721575,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007256877608597279,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.006779699586331844,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.0048655178397893906,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.17.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.2307678759098053,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.21631067991256714,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.21128687262535095,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.19153712689876556,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10823165625333786,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.10287821292877197,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12228357791900635,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11200681328773499,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.1097843274474144,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.09722528606653214,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.09243679791688919,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.062156498432159424,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.053487055003643036,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05176641792058945,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.05135784298181534,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.031021732836961746,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.026394937187433243,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.026251936331391335,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.024122677743434906,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.023860177025198936,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.016037827357649803,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015650901943445206,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01531386747956276,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.009858328849077225,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.17.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.18570582568645477,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1642397940158844,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.15666881203651428,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.13476502895355225,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.08618844300508499,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.07797126471996307,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.10130489617586136,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.09261273592710495,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.0889883041381836,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07218995690345764,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.06560887396335602,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.052280109375715256,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.04535114765167236,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04246007278561592,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.041766222566366196,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.026546914130449295,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.023625262081623077,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.023421382531523705,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.021107539534568787,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.020694827660918236,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.015236389823257923,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.016874440014362335,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.014392996206879616,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.013600500300526619,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.17.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.17736780643463135,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.16700851917266846,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.16366377472877502,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.14899642765522003,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.08359556645154953,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07985702157020569,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.0931326299905777,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.08601079136133194,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.08471163362264633,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.0755840316414833,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.07199624180793762,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.047673650085926056,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.04129962995648384,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.04021883010864258,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.039961546659469604,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.023864155635237694,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02101775072515011,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.02094459906220436,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.019434548914432526,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.019275497645139694,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.01285453699529171,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.01319224014878273,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.012511652894318104,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009360500611364841,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.17.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.24139344692230225,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.22740350663661957,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.22301454842090607,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.20315507054328918,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.11351452767848969,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10851236432790756,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.1263357400894165,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.11657419055700302,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.11499345302581787,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.1026596948504448,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09772476553916931,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06430108100175858,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05568019673228264,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.05429593473672867,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05396902188658714,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.032091207802295685,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.027730753645300865,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.02763538248836994,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02550455741584301,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02529572695493698,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.016790315508842468,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.0164338368922472,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.01631869561970234,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.01049888413399458,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.17.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.24254877865314484,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.21998032927513123,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.21172241866588593,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.18895478546619415,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.11194678395986557,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.10345654934644699,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.12957748770713806,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11905865371227264,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.11503962427377701,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09790395200252533,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.0921463817358017,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06633715331554413,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05740034207701683,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.05415971204638481,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.053389839828014374,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.033398743718862534,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.028915051370859146,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.028749898076057434,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.026232372969388962,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.025759469717741013,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.018417922779917717,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.01920083351433277,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.017392221838235855,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.014276823960244656,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.18.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.10879064351320267,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1016271710395813,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.0985458716750145,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.08911555260419846,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05070674046874046,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.0477847158908844,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.057999223470687866,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05359775200486183,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05146189033985138,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04546910896897316,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04319994896650314,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.02952215075492859,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02564968355000019,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.024352051317691803,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.024039650335907936,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.014757988974452019,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.012668078765273094,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.012539160437881947,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.011631435714662075,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.011438938789069653,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007788323797285557,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007974749431014061,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.00735617708414793,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005477437749505043,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.18.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.09424494206905365,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.088041752576828,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.08501125872135162,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.0769127756357193,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.043843306601047516,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04116395488381386,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.050745464861392975,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.04691237211227417,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.04451471567153931,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.039343707263469696,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.037421125918626785,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.025709262117743492,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.022400060668587685,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02103750966489315,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.02069343999028206,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.012872682884335518,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.010882055386900902,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.010733929462730885,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.009979810565710068,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.009772653691470623,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.006771084852516651,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.006805945187807083,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.0063411216251552105,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.0045417496003210545,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.18.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.22476796805858612,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.2106444388628006,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.20545195043087006,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.18619906902313232,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10525499284267426,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.09984651952981949,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11910408735275269,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.1096416786313057,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10679791867733002,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.09458810091018677,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.08975309133529663,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06051240861415863,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.0523713082075119,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05033421516418457,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04986567050218582,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03019687905907631,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.025704199448227882,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.025524457916617393,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.023520592600107193,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.023213578388094902,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.015636317431926727,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015353994444012642,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01488763839006424,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.00969889760017395,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.18.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.21289214491844177,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1841692328453064,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.174070805311203,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.14662490785121918,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.09818492829799652,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.08732353150844574,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11557363718748093,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10606444627046585,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10202129930257797,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07985175400972366,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.07201198488473892,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.0597214512526989,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05198816582560539,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04840991646051407,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04754909127950668,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.030335603281855583,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02697843126952648,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.026724446564912796,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.023685742169618607,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02314230054616928,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.017391610890626907,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.019322741776704788,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.016331186518073082,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.015549018047749996,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.18.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.1748240441083908,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.16465935111045837,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.1612785905599594,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.14692528545856476,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.08235965669155121,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07866301387548447,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.09169411659240723,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.08475528657436371,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.08341901004314423,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.07448887079954147,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.07097543030977249,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.046865541487932205,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.040744949132204056,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.03965839743614197,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.03941004350781441,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.023454055190086365,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.020830277353525162,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.02074909396469593,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.01928732544183731,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.01913093961775303,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.012619871646165848,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.013210343196988106,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.012268878519535065,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009525515139102936,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.18.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.235576793551445,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.22201204299926758,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.2177143692970276,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1983848363161087,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.11065399646759033,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10579506307840347,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.12307111918926239,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.11361894756555557,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.11208515614271164,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.1001298576593399,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.0952560231089592,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06250341981649399,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05424153804779053,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.052896857261657715,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.052575744688510895,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.031182028353214264,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.026969078928232193,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.026870064437389374,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.024807199835777283,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02460337057709694,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.016229869797825813,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.015913326293230057,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.015771999955177307,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.01006526593118906,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.18.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.24320296943187714,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.21946334838867188,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.21073360741138458,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.18737004697322845,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.11203669011592865,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.10313169658184052,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.12983664870262146,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.1194329783320427,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.11527380347251892,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09730948507785797,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.09143362194299698,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06653548777103424,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.0574934147298336,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.054198041558265686,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.05338568240404129,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.033667948096990585,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.028865886852145195,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.028706280514597893,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.026052754372358322,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.025551844388246536,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.018863778561353683,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.01911727711558342,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.017864946275949478,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.014125585556030273,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.19.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.10509544610977173,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.09827423840761185,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.09529402107000351,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.08625444769859314,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.04898308590054512,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04613293707370758,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.056159280240535736,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05187027528882027,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.04969123750925064,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.043960608541965485,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04177046939730644,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.02853114902973175,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02480371668934822,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.023489514365792274,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.023173945024609566,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.014264038763940334,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.012165384367108345,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.012026929296553135,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.011162776499986649,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.010964769870042801,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007495549973100424,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007584757171571255,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007050846237689257,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005108269862830639,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.19.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.09133706986904144,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.08545225858688354,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.0824800506234169,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.0746617540717125,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.04248495027422905,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.03989851847290993,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.04918800666928291,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.04553355276584625,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.043116018176078796,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.0381731241941452,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.036328017711639404,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.024939995259046555,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.021732067689299583,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.020375298336148262,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.020046664401888847,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.012468907982110977,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.010524876415729523,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.010376677848398685,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.009658783674240112,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.00945277139544487,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.0065405238419771194,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.006566834636032581,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.0061128828674554825,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004349882248789072,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.19.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.2180389165878296,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.204375758767128,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.19925588369369507,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.18056806921958923,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10197015851736069,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.09671933948993683,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.1157430112361908,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10632819682359695,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10345117002725601,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.09169856458902359,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.08707123249769211,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.058761294931173325,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05076676234602928,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04880377650260925,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04832632094621658,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.029328251257538795,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.024927867576479912,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02474508248269558,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.022815031930804253,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.022507091984152794,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.015184150077402592,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01491488702595234,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.014435616321861744,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.009456673637032509,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.19.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.2028786838054657,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1768070012331009,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.16390769183635712,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.14063484966754913,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.09357460588216782,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.0814286395907402,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11714062094688416,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10693077743053436,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.09741123765707016,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07753180712461472,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.07246294617652893,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.060517460107803345,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.051978230476379395,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04587804898619652,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04434844106435776,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.030423497781157494,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.025178493931889534,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.024563167244195938,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.02221992239356041,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.021252483129501343,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.016809390857815742,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.018034521490335464,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.014841427095234394,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.013813905417919159,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.19.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.17070958018302917,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.16078247129917145,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.15743492543697357,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.14347001910209656,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.08030395209789276,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07670028507709503,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.08955137431621552,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.08277027308940887,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.0813739150762558,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.07272409647703171,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.06926751881837845,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.04578506946563721,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.039794474840164185,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.0386941134929657,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.038443222641944885,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.022921089082956314,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.020354432985186577,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.02027423307299614,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.018863270059227943,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.018707934767007828,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.012369566597044468,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.01295726839452982,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.012016991153359413,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009386149235069752,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.19.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.22832819819450378,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.21517916023731232,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.2110188752412796,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.19225245714187622,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10715781897306442,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10244233906269073,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11924026906490326,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.1100943386554718,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.10851656645536423,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09701696038246155,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09228760004043579,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06054231524467468,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.052556563168764114,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.05123080685734749,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05091516673564911,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.03019244782626629,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.026149163022637367,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.026053840294480324,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02406574971973896,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.023867681622505188,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.015667377039790154,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.015481269918382168,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.015203375369310379,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009851770475506783,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.19.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.2435864359140396,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.2194809466600418,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.21047478914260864,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.18678614497184753,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.11233092844486237,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.10310163348913193,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.13067443668842316,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.12010639160871506,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.11563616991043091,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09730764478445053,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.09124372899532318,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06700566411018372,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.057863786816596985,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.054333530366420746,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.053473394364118576,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.033809907734394073,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.028922490775585175,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.028748879209160805,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.026056719943881035,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.025522667914628983,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.018835285678505898,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.019176695495843887,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.0177595317363739,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.014124304987490177,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.20.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.10903516411781311,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.10199686139822006,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.0990505963563919,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.08961862325668335,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.050730712711811066,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04789204150438309,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05792080610990524,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.053520869463682175,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05149059370160103,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04557343199849129,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04329073056578636,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.0294171255081892,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.025561358779668808,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.024304164573550224,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.02400035411119461,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.01469616312533617,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.012523341923952103,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.012395222671329975,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.011481828056275845,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.01129187736660242,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007688191719353199,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007704257033765316,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007263535168021917,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005077126435935497,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.20.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.09323921799659729,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.0872713178396225,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.08441770821809769,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.07646411657333374,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.043342188000679016,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.040762413293123245,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.049987711012363434,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.04628319293260574,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.043984271585941315,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.038994479924440384,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.03710097074508667,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.02532433159649372,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02208852767944336,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.020765654742717743,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.020447155460715294,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.012645448558032513,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.010703814215958118,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.010556155815720558,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.009820563718676567,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.009620515629649162,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.006617685314267874,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.006630907766520977,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.006206777412444353,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004344025626778603,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.20.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.21998469531536102,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.20628130435943604,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.2011307179927826,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.18234199285507202,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10292329639196396,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.09758885949850082,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.1166112869977951,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10728335380554199,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.1043798103928566,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.09255841374397278,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.0878346711397171,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.059271588921546936,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05132820084691048,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.049261271953582764,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04876718297600746,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.029630720615386963,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.025221766903996468,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.025019973516464233,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.023110851645469666,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.022794663906097412,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.015426227822899818,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01520472764968872,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.014680837281048298,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.009770258329808712,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.20.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.2135217785835266,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1876457780599594,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.1790478229522705,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.15509666502475739,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.09883054345846176,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.08931993693113327,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11560370773077011,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10577316582202911,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10224642604589462,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.08233555406332016,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.07634363323450089,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.05949946865439415,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05156583711504936,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04840240254998207,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04763028025627136,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03003716468811035,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.026588907465338707,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.026379186660051346,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.023579156026244164,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02311786822974682,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.016913754865527153,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.018598800525069237,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.015953615307807922,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.014688062481582165,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.20.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.1709129363298416,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.16101884841918945,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.15773896872997284,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.14373506605625153,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.08041058480739594,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07684381306171417,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.08962429314851761,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.08284296095371246,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.08146647363901138,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.07284395396709442,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.06948812305927277,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.04579346626996994,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.03983600065112114,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.03875928372144699,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.03850594535470009,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.022921333089470863,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02041970007121563,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.02034006454050541,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.018936146050691605,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.018786514177918434,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.012354549951851368,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.013042952865362167,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.012003975920379162,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009499986656010151,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.20.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.22757501900196075,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.21450594067573547,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.21035096049308777,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1916869580745697,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10676974058151245,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10213512927293777,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11880265176296234,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10969148576259613,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.10816723108291626,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09671048820018768,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09206898510456085,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.060390304774045944,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05234789475798607,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.05103025585412979,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05071893706917763,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.030120868235826492,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.026018036529421806,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.025919368490576744,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02394738234579563,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02374700829386711,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.01565638743340969,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.015351495705544949,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.01518993079662323,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.00970100611448288,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.20.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.24943308532238007,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.22431260347366333,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.2151603400707245,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.19046208262443542,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.11489902436733246,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.10538823157548904,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.13372839987277985,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.12261500954627991,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.11831611394882202,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.0991685763001442,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.09303814172744751,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06843115389347076,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05901316553354263,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.05551072210073471,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.054655276238918304,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.034538980573415756,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.02947293594479561,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.0293109193444252,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.02645593136548996,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.025924500077962875,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.01920294761657715,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.019426211714744568,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.018122024834156036,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.014222368597984314,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.21.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11662577092647552,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.10912372171878815,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10600439459085464,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09598373621702194,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05425721034407616,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.051281414926052094,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.06183452904224396,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05707455426454544,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05507407709956169,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.048773616552352905,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04635925218462944,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.0313933826982975,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.027315767481923103,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.026043685153126717,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.025740204378962517,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.015710782259702682,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.013529455289244652,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.013400502502918243,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.012433131225407124,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.01224235724657774,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.008279255591332912,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008460589684545994,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007856935262680054,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.0057881493121385574,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.21.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.09648173302412033,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.09026537835597992,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.08714485168457031,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.0788889154791832,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.04476480185985565,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.042059507220983505,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05181983858346939,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.04799339175224304,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.04541424661874771,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04026023671030998,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.0383649617433548,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.02625712752342224,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.022891350090503693,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.021457474678754807,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.021112946793437004,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.013128322549164295,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.011075740680098534,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.010916633531451225,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.010169686749577522,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.009948360733687878,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.006888191681355238,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.006895834114402533,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.006446207873523235,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004543342161923647,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.21.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.22415220737457275,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.210222989320755,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.2051772028207779,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.186041459441185,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10475669056177139,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.09945885837078094,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11842362582683563,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10886228084564209,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10625490546226501,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.09431686997413635,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.08954421430826187,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06019069626927376,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.052071463316679,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.050114504992961884,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04964923858642578,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.030073100700974464,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02564401365816593,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.025461236014962196,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.02351754531264305,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.023214275017380714,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01563115417957306,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015402172692120075,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.014892252162098885,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.00985969789326191,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.21.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.21583010256290436,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.19187171757221222,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.1835043877363205,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.15330038964748383,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10056887567043304,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.09161713719367981,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11706319451332092,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10725478082895279,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10364938527345657,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.0832715779542923,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.07423164695501328,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06068134680390358,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05220441147685051,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04913906008005142,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04839916154742241,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.030645422637462616,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.026639867573976517,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02642972394824028,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.023409564048051834,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02296171337366104,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.017217127606272697,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01822039484977722,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.016308287158608437,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.014029272831976414,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.21.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.1652001589536667,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.1556084156036377,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.15246954560279846,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1390163004398346,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.07774414867162704,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07426716387271881,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.08648594468832016,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.08002723008394241,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.07876813411712646,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.07040027529001236,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.06704747676849365,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.044199876487255096,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.03848053142428398,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.0374496690928936,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.03720841184258461,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.022116146981716156,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.01968015357851982,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.01960715278983116,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.018244313076138496,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.0180966854095459,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.011877423152327538,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.012503285892307758,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.011552260257303715,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009035015478730202,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.21.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.22594009339809418,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.21300296485424042,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.2089824676513672,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1905107945203781,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10599900037050247,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10142025351524353,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11778046935796738,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10880701243877411,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.10738224536180496,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.0960383266210556,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09140989929437637,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.05985467508435249,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05192295461893082,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.05065327510237694,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.050350990146398544,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.029853450134396553,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.025800717994570732,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.025708923116326332,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02373684011399746,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.023548195138573647,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.015508548356592655,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.01517653651535511,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.01506376825273037,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009535575285553932,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.21.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.2408285290002823,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.216168612241745,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.2066570371389389,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.1831388622522354,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.11069903522729874,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.10114524513483047,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.1303272247314453,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.1190127357840538,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.11410122364759445,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.0954665094614029,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.0896192416548729,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06629687547683716,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05720800906419754,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.0534781739115715,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.05257180333137512,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.033351168036460876,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.02841857075691223,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.028206629678606987,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.025497352704405785,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.024920852854847908,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.01828635111451149,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.018795128911733627,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.017096219584345818,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.013716635294258595,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.22.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.119473397731781,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.11194922775030136,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10890927910804749,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09881491959095001,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05561109259724617,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.05266039818525314,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.06320428848266602,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05831583961844444,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05638683959841728,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.050052765756845474,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04762373864650726,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.03206397220492363,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.027856258675456047,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.026620054617524147,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.026329180225729942,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.016017097979784012,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.013720489107072353,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.013596311211585999,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.012605882249772549,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.012416478246450424,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.00836938340216875,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008418033830821514,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007940366864204407,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005573504604399204,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.22.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.09747941792011261,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.09134180843830109,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.08838734775781631,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.08016544580459595,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.04525994881987572,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04262950271368027,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.052129410207271576,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.04824380204081535,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.045892857015132904,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04078720882534981,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.03885037451982498,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.026353729888796806,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.022993987426161766,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02167062647640705,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.021350065246224403,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.01316677127033472,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.011146198958158493,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.010999300517141819,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.010241041891276836,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.010038257576525211,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.006871154997497797,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.006869609002023935,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.006458579562604427,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004471750929951668,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.22.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.22421200573444366,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.21051962673664093,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.20542466640472412,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.18649610877037048,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10487627983093262,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.09955646842718124,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11885122954845428,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10928656905889511,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10631464421749115,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.09459607303142548,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.08997482061386108,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06038345396518707,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05230138078331947,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05016155540943146,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04964544624090195,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.030176810920238495,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.025631923228502274,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02542012929916382,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.023523760959506035,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.023197079077363014,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01569277234375477,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015364398248493671,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.014934941194951534,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.009745847433805466,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.22.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.21682922542095184,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1977822333574295,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.19164225459098816,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.16916774213314056,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.1011485829949379,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.09388624876737595,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.1158876121044159,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10567678511142731,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10325758159160614,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.08713367581367493,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.08170358836650848,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.05964751914143562,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.051493316888809204,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.049458399415016174,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04898485168814659,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.030353408306837082,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.026968369260430336,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.026825426146388054,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.0243397019803524,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.024035140872001648,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.017452340573072433,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01846771128475666,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01686069928109646,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.014510632492601871,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.22.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.16548378765583038,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.15584050118923187,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.1527305245399475,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.13922277092933655,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.07779058814048767,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07431747764348984,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.08651437610387802,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.08007966727018356,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.07878678292036057,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.07044731825590134,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.06710588932037354,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.044153984636068344,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.03843001276254654,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.03740822896361351,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.03716667741537094,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.022069202736020088,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.019569717347621918,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.019498154520988464,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.018119098618626595,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.017977992072701454,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.011785700917243958,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.012307249009609222,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.011452864855527878,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.008765824139118195,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.22.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.22458809614181519,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.21175573766231537,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.20768703520298004,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1894550919532776,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10535695403814316,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10081446915864944,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.117031030356884,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10814303904771805,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.10671176761388779,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09548664838075638,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09092625975608826,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.059419214725494385,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.051614925265312195,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.05034533143043518,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05005204305052757,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.029634464532136917,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.025648484006524086,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.02555929310619831,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02361512929201126,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02342325821518898,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.015364906750619411,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.0151000265032053,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.014925613068044186,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009501813910901546,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.22.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.24191829562187195,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.2162265181541443,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.20658139884471893,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.18313398957252502,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.11087814718484879,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.10097811371088028,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.1299862414598465,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11925014108419418,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.11444813013076782,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09531968832015991,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.089540496468544,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06633064150810242,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05717924237251282,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.05338979512453079,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.05247656628489494,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03342391178011894,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.028078727424144745,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.027888484299182892,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.02505379728972912,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.02447020635008812,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.018334584310650826,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.018221480771899223,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.01715649850666523,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.012898706831037998,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.23.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11622966825962067,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.10902583599090576,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10606148838996887,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09626828134059906,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.054156314581632614,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.05131960287690163,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.06152050197124481,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05678265169262886,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05489759519696236,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04882807657122612,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04646727815270424,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.031233523041009903,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.027127526700496674,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.025928398594260216,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.0256439708173275,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.015607230365276337,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.013338545337319374,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.013220611959695816,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.01226375438272953,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.012083454057574272,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.008161816745996475,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008152423426508904,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007759529631584883,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005350823048502207,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.23.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.09751016646623611,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.09137767553329468,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.08849240094423294,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.08024993538856506,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.045285262167453766,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04269370436668396,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05198816582560539,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.048169758170843124,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.04593311622738838,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.040820907801389694,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.0388643704354763,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.026309410110116005,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.022970274090766907,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.021680643782019615,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.02136838436126709,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.013147716410458088,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.011160776019096375,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.01102105900645256,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.010266616009175777,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.010067474097013474,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.006870684213936329,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.006888683419674635,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.006470184773206711,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004510523751378059,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.23.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.23017968237400055,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.21640925109386444,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.21138739585876465,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.19208276271820068,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10778962075710297,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.10255508124828339,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12181157618761063,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11195176094770432,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10926466435194016,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.0973164364695549,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.09260430932044983,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.061879776418209076,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05353068932890892,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.051579758524894714,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.051098115742206573,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03091343492269516,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.026343178004026413,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.026159146800637245,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.02419242076575756,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.023893950507044792,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.016070669516921043,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015742624178528786,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01534010749310255,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.009989838115870953,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.23.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.21387602388858795,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.19380104541778564,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.185722216963768,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.16734835505485535,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.09980098903179169,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.09147027879953384,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11709446460008621,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10732121765613556,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10254930704832077,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.08714273571968079,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.08249481767416,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06033892184495926,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.0518009327352047,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04832179844379425,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04748028144240379,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.030279934406280518,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.025533808395266533,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02521923929452896,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.023044664412736893,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02252216637134552,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.016377843916416168,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.016738884150981903,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01524729561060667,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.01203305646777153,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.23.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.1656322181224823,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.15610846877098083,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.1529693603515625,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1394125521183014,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.07794100046157837,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07450380176305771,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.08663588017225266,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.08017586171627045,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.07897118479013443,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.07061201333999634,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.06729492545127869,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.04422306269407272,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.03852066397666931,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.0375225804746151,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.03729259595274925,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.022123204544186592,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.01969006657600403,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.019622089341282845,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.018246924504637718,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.018103448674082756,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.011850805953145027,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.012468324974179268,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.011528898030519485,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.008977427147328854,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.23.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.22831383347511292,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.2152879685163498,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.21125072240829468,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.19269368052482605,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.1071975901722908,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10256428271532059,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11901352554559708,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10999200493097305,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.10858415067195892,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09714939445257187,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09249847382307053,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06043856590986252,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.052491556853055954,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.051232289522886276,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05092524364590645,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.03014613874256611,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.026122406125068665,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.026031289249658585,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.024052446708083153,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.023868052288889885,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.01566264219582081,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.015412019565701485,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.01522883865982294,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.00975710991770029,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.23.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.23722773790359497,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.21157753467559814,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.20140281319618225,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.17835137248039246,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.1087513118982315,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.09859037399291992,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.12860967218875885,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11806235462427139,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.11250893026590347,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09335780888795853,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08787814527750015,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.0657648965716362,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05672069266438484,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.05251653119921684,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.0514850877225399,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03323495388031006,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.027798226103186607,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.027546830475330353,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.02482239156961441,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.024164892733097076,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.018362808972597122,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.018352914601564407,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.017048612236976624,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.013177787885069847,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.24.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.1232275515794754,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.11561839282512665,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.11266712099313736,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.10223440080881119,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05744120106101036,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.054509714245796204,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.0650893896818161,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.06005315110087395,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.058257389813661575,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.05181814730167389,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.049357980489730835,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.033094264566898346,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02871972881257534,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.027542173862457275,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.027260232716798782,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.01654861681163311,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.014246920123696327,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.014134161174297333,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.013121926225721836,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.012945365160703659,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.008697157725691795,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.00880153477191925,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.008310128003358841,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005933419801294804,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.24.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.10244925320148468,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.09605041146278381,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.09314653277397156,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.08442787826061249,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.04761253669857979,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.044965460896492004,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05444461852312088,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05046187341213226,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.04825792461633682,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.042912475764751434,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04083546996116638,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.027585426345467567,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.024064062163233757,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.022793058305978775,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.022482315078377724,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.013779280707240105,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.011732801795005798,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.011598690412938595,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.01078894454985857,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.010596754029393196,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007209200412034988,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007221677806228399,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.006816305220127106,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004738957621157169,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.24.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.23600363731384277,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.22197560966014862,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.2169809490442276,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.19734448194503784,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.11057785153388977,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.1053362712264061,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.1244315579533577,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11455591768026352,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.11203976720571518,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.09986662864685059,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.09507019072771072,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06321312487125397,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.054718900471925735,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05287632718682289,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.05241744965314865,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.031540803611278534,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.026961689814925194,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.026783639565110207,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.024759309366345406,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.024475164711475372,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01630423776805401,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.016004983335733414,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.015615434385836124,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.010068265721201897,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.24.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.21544340252876282,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.19746752083301544,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.19182869791984558,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.16817539930343628,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.09999421238899231,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.09327526390552521,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11470124125480652,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10387975722551346,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10197778791189194,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.08629787713289261,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.07972521334886551,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.058960121124982834,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.050727203488349915,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04898643493652344,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.048590369522571564,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03019978664815426,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.0269321221858263,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.026837538927793503,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.02439483441412449,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02413998357951641,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01761212944984436,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01861497014760971,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.017109358683228493,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.014869218692183495,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.24.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.16211648285388947,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.15266495943069458,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.14960050582885742,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.13645200431346893,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.07623868435621262,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07284871488809586,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.08481206744909286,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.07841122895479202,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.0772370919585228,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.06902992725372314,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.06584184616804123,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.04322124645113945,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.037647977471351624,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.036676447838544846,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.0364437997341156,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.021635832265019417,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.019195158034563065,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.01912677474319935,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.017772261053323746,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.01763143017888069,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.01156865805387497,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.012087925337255001,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.011254983954131603,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.00862080231308937,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.24.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.22478261590003967,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.21183529496192932,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.20785261690616608,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.18963965773582458,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10545721650123596,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10088086128234863,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11712393164634705,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10816416144371033,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.10680117458105087,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09552549570798874,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09103672206401825,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.05948382616043091,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05163608491420746,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.0503927506506443,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05009988322854042,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.02972906082868576,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.025696823373436928,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.025609202682971954,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.023652734234929085,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02346635051071644,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.01552775502204895,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.015154335647821426,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.015108958818018436,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.00959530845284462,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.24.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.2309272140264511,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.20732103288173676,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.19856074452400208,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.17641662061214447,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.10581234842538834,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.09682034701108932,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.12331025302410126,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11335767060518265,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.1090892031788826,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.0914616510272026,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08604372292757034,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.0629652589559555,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.054349061101675034,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.050892267376184464,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.05006208270788193,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03165483474731445,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.026705751195549965,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.02652757801115513,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.02390461042523384,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.02336304634809494,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.017222702503204346,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.01722107082605362,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.016122426837682724,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.012090994976460934,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.25.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.1188727468252182,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.11161104589700699,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10867322236299515,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09874430298805237,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05537440627813339,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.052513789385557175,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.06289292126893997,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05800836160778999,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05614986643195152,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.05000294744968414,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04772007465362549,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.03195131942629814,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.027716342359781265,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.026517687365412712,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.026227932423353195,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.015966227278113365,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.013642469421029091,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.013526445254683495,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.012559217400848866,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.012380038388073444,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.008339639753103256,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008333646692335606,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007941028103232384,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005480543710291386,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.25.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.0989796370267868,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.09287092089653015,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.08999050408601761,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.08161403983831406,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.04598936066031456,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.043388478457927704,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05284051597118378,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.048915501683950424,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.04664989560842514,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.041484393179416656,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.039552945643663406,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.026755308732390404,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.023341728374361992,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02202519215643406,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.021708231419324875,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.01336866058409214,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.011355218477547169,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.011216692626476288,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.010443372651934624,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.010250462219119072,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007001611404120922,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.0070218914188444614,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.006598575972020626,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004621368832886219,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.25.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.23091191053390503,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.21716322004795074,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.21246185898780823,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.19337181746959686,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10812485963106155,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.1030648872256279,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12161962687969208,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.1118607223033905,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10957806557416916,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.09771163761615753,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.0931093692779541,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06177058443427086,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.0534069761633873,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05167582631111145,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.05127005651593208,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.030810730531811714,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.026342911645770073,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02619207836687565,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.02419852837920189,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.0239435862749815,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01592155732214451,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01559802982956171,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01525444258004427,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.00980640109628439,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.25.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.21286626160144806,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.19161009788513184,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.1847781538963318,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.15398718416690826,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.0995679423213005,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.09189394116401672,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.1137842983007431,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10416989028453827,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.1018432155251503,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.08248303830623627,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.0734061598777771,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.058553546667099,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05051610246300697,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.0484115406870842,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04793354123830795,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.029440131038427353,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.025983192026615143,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02583054080605507,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.02272535115480423,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.022418392822146416,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.016302457079291344,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.017289824783802032,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.0156718660145998,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.013111352920532227,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.25.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.16706611216068268,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.15741890668869019,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.1543402373790741,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.14080846309661865,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.07855116575956345,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.0751175731420517,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.08728044480085373,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.0807100459933281,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.0795883908867836,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.07116147875785828,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.06784701347351074,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.04447893425822258,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.038710806518793106,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.037744566798210144,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.037515632808208466,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.02228545770049095,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.01967003382742405,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.019601808860898018,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.018203478306531906,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.018059441819787025,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.0119312210008502,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.012255227193236351,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.011620404198765755,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.008590448647737503,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.25.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.22784003615379333,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.21492807567119598,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.210894376039505,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.19250692427158356,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10689293593168259,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10233408957719803,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11880436539649963,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10960544645786285,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.10826438665390015,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09689070284366608,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09241950511932373,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06027375906705856,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.052325744181871414,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.051098357886075974,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.050802476704120636,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.0301650483161211,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.026083000004291534,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.025998283177614212,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02403143420815468,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.023847684264183044,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.01586478389799595,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.015420272015035152,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.015456105582416058,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009817397221922874,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.25.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.2336633950471878,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.20944860577583313,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.2003379911184311,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.1779487580060959,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.10696540027856827,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.09766380488872528,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.12463634461164474,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11490686982870102,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.1104070171713829,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09238935261964798,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08687426894903183,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06354502588510513,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05517936497926712,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.051543574780225754,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.05062621459364891,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03192100673913956,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.027121499180793762,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.026927027851343155,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.02429022639989853,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.023725301027297974,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.01709550805389881,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.017629951238632202,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.015900244936347008,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.01251482404768467,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.26.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11825139820575714,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.11116188019514084,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10832415521144867,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09852086752653122,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.055159710347652435,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.052369918674230576,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.06258121877908707,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05769575014710426,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05589371174573898,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04987591132521629,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.047646619379520416,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.03180104121565819,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.027554091066122055,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02639719657599926,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.026121152564883232,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.01589849591255188,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.013572999276220798,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.013461284339427948,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.012503326870501041,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.012329187244176865,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.008343510329723358,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008271344937384129,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007956434041261673,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005426438990980387,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.26.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.10112487524747849,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.09500937908887863,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.0921650305390358,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.08379633724689484,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.04704521223902702,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04449118673801422,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.053797125816345215,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.049787264317274094,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.047692205756902695,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.0425218865275383,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.040570296347141266,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.027247034013271332,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02374294027686119,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02251184731721878,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.022220129147171974,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.01361843291670084,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.011570786125957966,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.011440226808190346,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.010656268335878849,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.010470432229340076,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007108927238732576,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007090247701853514,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.0067278011702001095,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.00461800629273057,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.26.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.23302631080150604,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.21939043700695038,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.21478186547756195,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.1956029236316681,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10916744917631149,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.10416367650032043,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12235777080059052,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11268837004899979,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.1106073260307312,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.09878078103065491,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.0941319391131401,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06211480498313904,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.053767114877700806,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05214064195752144,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.05175069719552994,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.030980607494711876,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.026475444436073303,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02634270489215851,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.024330424144864082,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.024089472368359566,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.015926668420433998,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015515675768256187,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.015301654115319252,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.009543677791953087,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.26.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.22585049271583557,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.20410026609897614,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.19740059971809387,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.16760694980621338,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10591736435890198,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.09816139936447144,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11998868733644485,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11008106172084808,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10806486010551453,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.08881014585494995,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.07995782792568207,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06185483559966087,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05352628231048584,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.051635630428791046,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.05120314657688141,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.0312529094517231,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02791881188750267,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.027789540588855743,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.02468666061758995,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.024408750236034393,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01758539117872715,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.018786463886499405,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.017023315653204918,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.014541474170982838,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.26.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.17162838578224182,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.1617683619260788,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.1586073935031891,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1446707546710968,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.08066519349813461,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07714658230543137,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.08946921676397324,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.08280643820762634,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.08172084391117096,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.07309775799512863,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.06965798139572144,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.04560656100511551,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.039679959416389465,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.038713499903678894,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.0384821891784668,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.022767940536141396,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.020104898139834404,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.0200339425355196,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.018594825640320778,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.018450424075126648,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.01204485073685646,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.01241946779191494,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.011727284640073776,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.008608908392488956,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.26.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.23180916905403137,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.21868181228637695,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.21457453072071075,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.19588914513587952,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.1087786927819252,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10415291041135788,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.12043467164039612,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.1115182489156723,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.11016535013914108,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09865272790193558,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.093961700797081,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.061166197061538696,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05320170521736145,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.05196162313222885,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05167778581380844,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.03049756959080696,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.026456071063876152,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.026373935863375664,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02436690591275692,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.024179819971323013,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.015773994848132133,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.015541463159024715,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.015352659858763218,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.00976306851953268,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.26.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.23356391489505768,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.20919770002365112,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.19981148838996887,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.17762495577335358,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.10682164132595062,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.09743095934391022,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.12519076466560364,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11533409357070923,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.11035633087158203,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09233520925045013,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08690385520458221,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06370227783918381,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05519343167543411,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.05135733634233475,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.05043230205774307,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03188973665237427,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.026862140744924545,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.02664831280708313,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.0240170955657959,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.02340601198375225,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.017020951956510544,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.01726696453988552,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.01577538251876831,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.01194158848375082,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.27.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11592917889356613,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.10886388272047043,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10590566694736481,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09639277309179306,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05398699268698692,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.051221296191215515,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.06153382360935211,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05671634152531624,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05476177856326103,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.048831529915332794,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.0467393733561039,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.0312773659825325,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02710692025721073,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02589510753750801,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.025604354217648506,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.015654046088457108,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.013383551500737667,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.01325880829244852,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.012343431822955608,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.012162288650870323,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.008235524408519268,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.00826953910291195,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007839464582502842,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005539952777326107,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.27.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.09999340027570724,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.09389979392290115,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.09094315767288208,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.08264204114675522,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.04649106040596962,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04386919364333153,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05343786999583244,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.04948408156633377,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.04713711515069008,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.042007170617580414,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04007219523191452,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.027060046792030334,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02359921485185623,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.022266309708356857,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.021944651380181313,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.013529465533792973,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.011468324810266495,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.011323428712785244,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.01056191511452198,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.010362914763391018,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007080812938511372,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.00707708764821291,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.006673534866422415,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.0046424055472016335,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.27.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.2317732870578766,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.21814841032028198,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.21341173350811005,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.19447506964206696,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10858810693025589,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.10353342443704605,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12195968627929688,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11224746704101562,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.11004147678613663,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.09824147075414658,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.09359737485647202,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.061929624527692795,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05356363207101822,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05188041180372238,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.0514773465692997,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03087109886109829,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02639692835509777,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.026255691424012184,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.024259483441710472,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02400430478155613,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.015910571441054344,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015559026040136814,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01526420284062624,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.00969763658940792,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.27.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.21339187026023865,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1878596544265747,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.17929866909980774,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.15429529547691345,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.09899552911520004,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.08929616957902908,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11507605761289597,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.1049758642911911,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10197747498750687,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.0819540023803711,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.07507383823394775,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.05926141515374184,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05101385340094566,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04837135970592499,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04773709177970886,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03008430451154709,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.026338044553995132,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.026168055832386017,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.023170851171016693,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.022765185683965683,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.017133012413978577,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.018079319968819618,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.016335878521203995,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.014094402082264423,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.27.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.17695380747318268,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.1667257398366928,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.1636105477809906,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1493685096502304,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.08312394469976425,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07955813407897949,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.0923800989985466,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.08532639592885971,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.08421700447797775,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.07537318766117096,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.07195214927196503,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.04702000319957733,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.04091740772128105,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.03993247449398041,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.03970177844166756,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.023549573495984077,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.020808562636375427,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.020742855966091156,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.01926250010728836,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.019119244068861008,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.012600621208548546,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.012950052507221699,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.01228317990899086,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.00909140519797802,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.27.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.2322900891304016,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.21915927529335022,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.21505552530288696,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.19639652967453003,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10900920629501343,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10437730699777603,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.12083600461483002,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.11175134032964706,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.11042255163192749,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09887584298849106,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09423281252384186,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.061353765428066254,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.053358614444732666,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.052118316292762756,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05181954428553581,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.030679574236273766,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02660377137362957,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.02651861310005188,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02452697604894638,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02434094436466694,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.01607484742999077,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.01571708731353283,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.015667837113142014,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.010017193853855133,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.27.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.23270876705646515,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.20760300755500793,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.1975930631160736,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.17574059963226318,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.10631130635738373,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.09644578397274017,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.12508651614189148,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11540217697620392,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.10993079841136932,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09153015911579132,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08622822910547256,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06365722417831421,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05515193194150925,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.05107679218053818,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.05007724463939667,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03183376416563988,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.026612918823957443,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.026380175724625587,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.023707987740635872,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.023066382855176926,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.016927404329180717,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.017037291079759598,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.015610570088028908,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.011624734848737717,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.28.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11263174563646317,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.10574226826429367,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10284187644720078,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09350556135177612,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.052403442561626434,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.049640338867902756,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05971691012382507,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05512787029147148,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.053133998066186905,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04733669012784958,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04522585868835449,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.03028734028339386,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.026298288255929947,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.025070467963814735,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.02477467805147171,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.015143228694796562,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.012880075722932816,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.01275092177093029,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.011859598569571972,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.01167372241616249,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007927692495286465,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007855046540498734,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007527327165007591,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005111608654260635,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.28.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.09780300408601761,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.09175994247198105,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.08879827708005905,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.08067218214273453,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.045412566512823105,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04283005744218826,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05230087414383888,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.048402514308691025,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.04605880752205849,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.0410228967666626,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.03915470466017723,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.026480013504624367,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.023097621276974678,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.021746644750237465,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.021428583189845085,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.01323673315346241,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.01119787897914648,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.011048853397369385,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.010310186073184013,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.010105956345796585,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.006932985968887806,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.006913719233125448,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.006520646158605814,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004518435336649418,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.28.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.22809696197509766,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.21450383961200714,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.2098034769296646,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.19096751511096954,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10661806911230087,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.1015460193157196,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11999785900115967,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11056061834096909,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10803799331188202,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.09639903157949448,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.09185008704662323,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.060905344784259796,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05272603780031204,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05092708766460419,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.050502583384513855,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03036980889737606,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.025916513055562973,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.025759341195225716,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.023818759247660637,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02354520559310913,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01566031575202942,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015314221382141113,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01499019842594862,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.009516580030322075,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.28.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.2192210555076599,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1918259710073471,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.18205052614212036,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.15244032442569733,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10208602249622345,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.09129483252763748,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11892940104007721,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10916823148727417,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10518627613782883,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.08278399705886841,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.07437398284673691,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06148388609290123,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.053128354251384735,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04985768347978592,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04908345639705658,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.031189020723104477,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.027155296877026558,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.026912668719887733,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.023574138060212135,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02308640256524086,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01766817271709442,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01872716099023819,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.016682742163538933,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.01455062534660101,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.28.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.17796555161476135,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.16777968406677246,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.16455283761024475,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1502961814403534,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.08356894552707672,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07996779680252075,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.09282436966896057,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.08575484901666641,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.08466242253780365,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.07578188925981522,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.07229388505220413,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.04725917801260948,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.04109402745962143,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.0400923490524292,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.03986287862062454,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.0236654169857502,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02082522213459015,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.020760899409651756,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.01927264966070652,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.019126052036881447,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.01262938603758812,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.012865976430475712,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.012312632985413074,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.008911791257560253,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.28.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.23044513165950775,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.2174312174320221,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.21339356899261475,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.19492796063423157,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10803020000457764,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10347706079483032,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11987311393022537,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.11078143119812012,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.10941867530345917,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09802128374576569,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09355884790420532,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06083780899643898,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05288359895348549,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.05165647342801094,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05135947838425636,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.030388399958610535,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.0263917725533247,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.026309393346309662,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02433915063738823,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02416270785033703,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.01586451567709446,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.015642838552594185,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.015452330932021141,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.0100252665579319,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.28.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.2382102906703949,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.21224868297576904,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.20221219956874847,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.17981171607971191,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.10884950309991837,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.09877683222293854,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.1285146027803421,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11784011125564575,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.11262001842260361,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09370075911283493,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08828599005937576,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06539227068424225,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.056461505591869354,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.052426643669605255,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.0514359213411808,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03279593586921692,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.02754933387041092,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.02732180990278721,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.024604609236121178,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.023963766172528267,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.017700403928756714,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.01791016198694706,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.01638508401811123,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.012588942423462868,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.29.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11420214176177979,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1072472557425499,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10430947691202164,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09476769715547562,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.0531684011220932,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.050368186086416245,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.06061474606394768,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.055890340358018875,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05391640216112137,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.048042427748441696,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.045897867530584335,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.03079068660736084,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.026693204417824745,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.0254654660820961,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.025162912905216217,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.015402666293084621,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.013108721002936363,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.012983664870262146,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.012073156423866749,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.011888762935996056,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.008088920265436172,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008031238801777363,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007692428305745125,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005288359709084034,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.29.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.09794040024280548,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.091907799243927,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.08882685005664825,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.0806802287697792,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.04545455053448677,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04279424622654915,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.0526287667453289,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.04866122454404831,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.04608236253261566,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.041042786091566086,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.0392007939517498,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.02663608081638813,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.0232161246240139,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02177828922867775,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.02143079787492752,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.01333216205239296,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.011243632063269615,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.011085311882197857,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.010366618633270264,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.010146976448595524,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007016903255134821,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007012613117694855,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.006582327652722597,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004636926576495171,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.29.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.2356460988521576,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.22163280844688416,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.21695025265216827,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.19760172069072723,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.11024944484233856,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.10512840747833252,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12365049123764038,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11386236548423767,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.11172069609165192,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.09969349205493927,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.09494107961654663,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06280877441167831,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.054309502243995667,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.0526362843811512,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.0522443987429142,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.031307414174079895,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.026761023327708244,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.026617178693413734,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.024577653035521507,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02432396449148655,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.016121655702590942,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015710989013314247,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.015476136468350887,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.009709177538752556,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.29.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.19932055473327637,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.17663027346134186,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.16955289244651794,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.1425412893295288,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.09229560196399689,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.08443345874547958,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.10574469715356827,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.09714020043611526,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.0948946475982666,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07625199854373932,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.0674336850643158,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.05456303805112839,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.047395866364240646,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.0452265664935112,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.044697705656290054,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.02767609804868698,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02486908622086048,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.024751245975494385,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.021949883550405502,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.021630126982927322,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.015783583745360374,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.017325691878795624,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.015135830268263817,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.013803888112306595,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.29.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.17194730043411255,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.16217058897018433,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.15907107293605804,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.14523857831954956,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.08070913702249527,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07724599540233612,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.08949484676122665,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.08279704302549362,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.08173475414514542,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.07317273318767548,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.06979316473007202,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.04554887115955353,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.039619408547878265,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.03866756707429886,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.038453228771686554,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.022745192050933838,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.020006902515888214,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.019938165321946144,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.018502678722143173,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.018360652029514313,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.012008031830191612,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.012244302779436111,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.011699189431965351,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.008343663066625595,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.29.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.22335723042488098,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.21078231930732727,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.20696274936199188,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.18899135291576385,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10471917688846588,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.100293830037117,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11590233445167542,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.1073014885187149,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.10603468120098114,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09501887112855911,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09058243781328201,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.05888533219695091,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05119044706225395,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.05000120773911476,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04972755163908005,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.029336055740714073,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.025438569486141205,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.025359485298395157,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02344236709177494,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02326439507305622,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.015165403485298157,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014902304857969284,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.014770164154469967,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009311341680586338,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.29.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.22572645545005798,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.20062196254730225,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.19066576659679413,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.16968169808387756,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.10280115157365799,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.09303400665521622,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.12188638746738434,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11197405308485031,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.10649129748344421,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.0885138288140297,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08357101678848267,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.062068160623311996,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05362845957279205,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.04951369762420654,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.048504747450351715,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03114606812596321,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.02602154202759266,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.025788262486457825,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.023239316418766975,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.0225977823138237,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.01687031239271164,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.016972271725535393,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.015554063022136688,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.011926230974495411,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.30.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11918476223945618,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.11213681846857071,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10934025794267654,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09962356835603714,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05555683746933937,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.0528067946434021,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.0627521499991417,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05808515101671219,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.056309182196855545,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.050343044102191925,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04799969121813774,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.03181920200586319,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.027713753283023834,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02658108063042164,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.026310011744499207,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.015902940183877945,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.013593673706054688,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.01348219346255064,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.012525869533419609,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.012352954596281052,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.008293220773339272,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008179343305528164,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007921269163489342,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005226477049291134,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.30.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.10615904629230499,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.09988604485988617,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.0971989706158638,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.08849401772022247,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.04945720359683037,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04692777991294861,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.0563713014125824,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.052059073001146317,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.050135787576436996,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.044808145612478256,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04282341152429581,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.028571151196956635,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.024813881143927574,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02365308813750744,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.023376189172267914,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.01425684429705143,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.012103677727282047,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.011982343159615993,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.011153707280755043,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.010980632156133652,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.0074133798480033875,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.0073090107180178165,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007040013559162617,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.0046585011295974255,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.30.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.23652546107769012,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.22300584614276886,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.21852420270442963,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.19936081767082214,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.1108437180519104,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.10595737397670746,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12375666946172714,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11411969363689423,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.11228113621473312,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.10045666247606277,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.09581831097602844,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06284129619598389,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05443448945879936,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05292150378227234,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.05255609005689621,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03132070600986481,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.026886478066444397,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.026762988418340683,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.024738984182476997,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.024507666006684303,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.016127677634358406,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.0157391969114542,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.015563271008431911,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.009713233448565006,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.30.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.22022448480129242,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.18475160002708435,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.1722017228603363,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.13447748124599457,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10147642344236374,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.08800078928470612,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11980702728033066,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11008824408054352,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10557706654071808,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07461044937372208,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.06805341690778732,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06173100695014,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05316154658794403,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.049260735511779785,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04830146208405495,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03103240765631199,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02633369155228138,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02605261281132698,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.021272901445627213,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02061423286795616,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.017143182456493378,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.017684398218989372,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.015941906720399857,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.013077387586236,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.30.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.17953136563301086,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.16934677958488464,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.16621416807174683,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1517501175403595,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.08427083492279053,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.0806688740849495,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.09336823225021362,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.08643221110105515,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.08537329733371735,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.07647339254617691,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.07292007654905319,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.04751088097691536,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.041350819170475006,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.040385451167821884,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.040152765810489655,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.02372768335044384,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.020871862769126892,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.020806152373552322,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.019301675260066986,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.019157618284225464,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.012503515928983688,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.012745598331093788,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.012186899781227112,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.008652571588754654,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.30.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.22918063402175903,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.2163591980934143,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.21242140233516693,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1940750628709793,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10742828994989395,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10290242731571198,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11914919316768646,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.11005621403455734,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.10878435522317886,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.0975077897310257,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09310683608055115,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06051676720380783,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.0525328554213047,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.051330097019672394,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05104218050837517,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.030232932418584824,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.026173053309321404,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.02609267644584179,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.024136336520314217,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.0239560529589653,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.01580970548093319,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.015428377315402031,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.015421816147863865,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.0097776735201478,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.30.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.22985254228115082,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.20483292639255524,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.19452212750911713,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.1732833832502365,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.10488693416118622,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.09488833695650101,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.12474998831748962,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11462242156267166,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.10865329951047897,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09055270999670029,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08545908331871033,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.0633130893111229,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.054952144622802734,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.05054868012666702,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.04946000501513481,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03172201290726662,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.02660508267581463,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.02632216550409794,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.023813508450984955,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.023117011412978172,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.01699465699493885,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.01741451770067215,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.015578373335301876,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.012226881459355354,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.31.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11791908740997314,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.11081194877624512,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10791081935167313,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09818435460329056,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05497179180383682,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.052124518901109695,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.062326353043317795,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05764251574873924,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05571839213371277,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04969467967748642,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.047379616647958755,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.0316028892993927,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.027518635615706444,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.026299264281988144,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.026000268757343292,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.015791485086083412,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.013491678051650524,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.013367218896746635,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.012422716245055199,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.01223401352763176,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.008250746876001358,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008187985047698021,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007859448902308941,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005293513182550669,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.31.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.10229875892400742,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.0961124524474144,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.09324668347835541,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.08490046858787537,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.047656577080488205,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04505304619669914,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05469273030757904,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.050531622022390366,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.048310186713933945,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04311005026102066,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.041184116154909134,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.02770409733057022,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02410569228231907,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02280881628394127,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.022497840225696564,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.013854792341589928,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.011709680780768394,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.011566905304789543,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.01078407559543848,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.010589172132313251,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007233802694827318,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007151266559958458,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.006822406779974699,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004619418643414974,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.31.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.23744137585163116,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.2235526293516159,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.21891333162784576,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.19950544834136963,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.11123760789632797,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.106178417801857,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12439054250717163,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11469164490699768,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.11271297931671143,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.10066098719835281,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.09597185254096985,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06317245215177536,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.0546836256980896,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05312032997608185,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.05274717137217522,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03149552643299103,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.026991967111825943,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02686469443142414,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.02479776367545128,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.024561993777751923,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.016197603195905685,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01582273840904236,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.015594861470162868,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.009769619442522526,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.31.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.20084325969219208,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1749046891927719,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.1670616865158081,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.1435975581407547,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.09327071160078049,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.08377977460622787,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.10636508464813232,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.09782665222883224,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.09605514258146286,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07605674862861633,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.06883663684129715,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.054687242954969406,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.04728129878640175,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04521391913294792,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04469958320260048,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.027501441538333893,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.024059003219008446,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02395394630730152,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.02074284665286541,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.020420406013727188,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.015089089050889015,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015794716775417328,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.014447878114879131,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.011728839948773384,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.31.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.18535137176513672,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.17483843863010406,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.1715688556432724,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1566728800535202,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.0870659202337265,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.08335229009389877,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.09645132720470428,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.08927749842405319,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.08817275613546371,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.07898228615522385,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.07526414841413498,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.049081094563007355,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.042707737535238266,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.041722651571035385,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04148934781551361,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.024502580985426903,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.021555418148636818,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.021487925201654434,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.019924037158489227,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.019781295210123062,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.012903401628136635,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.013145286589860916,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.012583097442984581,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.008915731683373451,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.31.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.23162713646888733,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.21861602365970612,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.21462741494178772,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.19606316089630127,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10862398892641068,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10404402762651443,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.12019655108451843,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.11131614446640015,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.11002146452665329,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09858427941799164,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09397587925195694,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06105375662446022,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05309300869703293,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.05188514292240143,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05160248652100563,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.030429834499955177,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.026403488591313362,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.026319975033402443,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.024322956800460815,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.024144457653164864,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.015748072415590286,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.015468493103981018,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.015345207415521145,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009675933979451656,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.31.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.23218275606632233,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.2070946842432022,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.19705830514431,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.1756436824798584,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.10588604211807251,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.09607494622468948,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.12560027837753296,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11533915996551514,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.10970161110162735,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09161553531885147,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08639257401227951,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06386744230985641,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05522824451327324,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.050950996577739716,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.04990160092711449,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.032217543572187424,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.026673678308725357,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.02641717530786991,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.023873625323176384,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.023197300732135773,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.01765684224665165,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.017261141911149025,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.01634099707007408,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.011930884793400764,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.32.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.1360718160867691,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1280682235956192,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.1248641312122345,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.11379780620336533,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.06367333233356476,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.06049247458577156,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.07219122350215912,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.06661171466112137,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.06451231986284256,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.05763997882604599,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.0551171749830246,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.03672640398144722,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.031850237399339676,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.030511382967233658,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.030193287879228592,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.01836380548775196,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.0156854335218668,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.01555663626641035,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.014463511295616627,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.014263969846069813,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.009594835340976715,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.009555963799357414,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.00915547739714384,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.006269590929150581,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.32.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11337392032146454,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.10663857311010361,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.1034589409828186,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09416703879833221,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05289973318576813,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.05003399774432182,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.060601308941841125,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.056116677820682526,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05362304672598839,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04788295924663544,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.045756980776786804,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.030716316774487495,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02679859660565853,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.025362012907862663,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.02500981278717518,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.015366654843091965,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.013072087429463863,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.01291469857096672,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.01205695141106844,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.011837635189294815,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.00805376935750246,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008074606768786907,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007611241191625595,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005324517376720905,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.32.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.2317042499780655,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.21842451393604279,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.21384784579277039,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.1950579285621643,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10863890498876572,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.10364515334367752,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12192738801240921,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11225181072950363,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.11000623553991318,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.09844548255205154,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.09391479194164276,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.061975013464689255,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.053596463054418564,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.051921334117650986,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.05152604728937149,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.030936511233448982,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02645256742835045,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.026311662048101425,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.024353640154004097,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.024099620059132576,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.015983978286385536,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015653351321816444,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.015338154509663582,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.009835836477577686,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.32.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11016008257865906,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.10394585877656937,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.1020064577460289,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09230871498584747,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.052486348897218704,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.0503266379237175,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.0586252324283123,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05374986305832863,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05313583463430405,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04734218865633011,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04481372609734535,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.030287999659776688,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.026093654334545135,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02551615983247757,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.025384357199072838,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.015185339376330376,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.013587174005806446,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.01355042401701212,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.012587150558829308,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.012507637031376362,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.0083035659044981,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.00885202456265688,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.008134787902235985,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.006664785090833902,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.32.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.15657125413417816,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.14767569303512573,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.14484019577503204,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.13229061663150787,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.07361336052417755,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.0704025849699974,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.08176401257514954,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.07558520138263702,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.07454704493284225,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.06677807867527008,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.0637054368853569,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.04173446446657181,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.03628852963447571,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.03539944067597389,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.03518633544445038,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.02094396762549877,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.018490206450223923,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.01843014918267727,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.017136573791503906,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.017007261514663696,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.011292667128145695,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.0115757891908288,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.011011835187673569,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.008197328075766563,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.32.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.21813590824604034,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.20594125986099243,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.20213215053081512,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.18464280664920807,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10228589177131653,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.09796948730945587,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11362709850072861,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10492010414600372,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.1035979613661766,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.0928751528263092,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.08861912786960602,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.05770661681890488,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05009249597787857,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.04892191290855408,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04863652214407921,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.028897617012262344,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02498859353363514,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.024904321879148483,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.023047588765621185,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.022864464670419693,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.015209242701530457,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014787109568715096,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.014825038611888885,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.00944529939442873,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.32.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.2116028070449829,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.18847046792507172,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.17856255173683167,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.15940463542938232,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.09664950519800186,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.08712083846330643,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.11522280424833298,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.10619613528251648,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.10013415664434433,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.0835053026676178,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.07908187061548233,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.05882548540830612,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05127086862921715,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.046838369220495224,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.04572630301117897,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.029674936085939407,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.02508104033768177,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.024780211970210075,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.02258821576833725,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.021905964240431786,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.016034502536058426,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.017003599554300308,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.014600585214793682,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.012529849074780941,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.33.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.12197361141443253,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.11495266109704971,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.11215932667255402,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.10234993696212769,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05703412741422653,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.05427283048629761,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.06456577032804489,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.059577684849500656,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.057775843888521194,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.05174429714679718,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.049533579498529434,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.03279206156730652,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.028483152389526367,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.027324410155415535,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.027044422924518585,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.016403278335928917,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.014069486409425735,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.013954966329038143,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.013005731627345085,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.012832802720367908,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.008606960065662861,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008602985180914402,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.008234039880335331,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005693930201232433,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.33.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.10365279018878937,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.09765322506427765,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.09501345455646515,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.08663515746593475,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.04836201295256615,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04587341845035553,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05512901023030281,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.050986889749765396,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.04899413883686066,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.043873801827430725,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04196621850132942,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.027940673753619194,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.024341028183698654,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.023160772398114204,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.022879978641867638,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.013974620029330254,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.011899867095053196,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.011774538084864616,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.010986248031258583,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.010811662301421165,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.00729422876611352,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007269533816725016,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.006922469474375248,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004730513319373131,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.33.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.2272699624300003,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.21451342105865479,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.21020233631134033,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.1919066458940506,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10646378993988037,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.10174019634723663,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11911961436271667,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10972603410482407,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10782015323638916,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.09660541266202927,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.09223415702581406,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.060447853058576584,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.052346520125865936,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.050839539617300034,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.050476789474487305,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03017343580722809,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02582015097141266,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.025698013603687286,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.02378639578819275,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.023556845262646675,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.015535992570221424,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01512252725660801,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.014960012398660183,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.009332514367997646,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.33.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.16376173496246338,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.15209269523620605,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.1485622078180313,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.13209307193756104,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.07725855708122253,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.07296770066022873,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.08651761710643768,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.07957950234413147,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.07851554453372955,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.06808467954397202,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.06332234293222427,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.044700272381305695,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.03877805173397064,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.037725381553173065,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.037477362900972366,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.022584106773138046,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02048100158572197,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.020417504012584686,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.01875295117497444,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.0186106376349926,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.0127799604088068,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.013840751722455025,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.012470496818423271,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.010849406011402607,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.33.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.15051652491092682,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.14204558730125427,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.1393755078315735,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1274143010377884,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.07068639248609543,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.06767521798610687,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.07834900915622711,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.0725250318646431,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.07160021364688873,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.06419216096401215,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.06127993389964104,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.0399666391313076,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.034771937876939774,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.03394940495491028,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.033748969435691833,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.019986649975180626,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.01765149086713791,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.01759403571486473,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.01635878160595894,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.01623760722577572,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.010618990287184715,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.010932542383670807,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.010349739342927933,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.007613704539835453,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.33.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.2162916511297226,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.20424304902553558,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.20058348774909973,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1833226978778839,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10141289979219437,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.0971466600894928,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11224967986345291,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10390687733888626,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.10267441719770432,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09212657064199448,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.08788150548934937,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.056992314755916595,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.0495791994035244,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.04844122752547264,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04817143455147743,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.02843667007982731,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02467736043035984,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.024601152166724205,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02276654727756977,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.022592300549149513,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.014727509580552578,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014504321850836277,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.014341315254569054,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.00914038997143507,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.33.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.21259130537509918,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.18972370028495789,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.179887592792511,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.16095605492591858,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.09691186249256134,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.08758421987295151,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.1156175285577774,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.10659684240818024,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.10046631097793579,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.08406703174114227,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.0796898901462555,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.0588652677834034,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05113603174686432,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.04676876962184906,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.04569871723651886,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.02959166280925274,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.02473282814025879,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.024448642507195473,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.02224319986999035,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.02156992442905903,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.0159979909658432,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.016379691660404205,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.014624346978962421,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.011668147519230843,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.34.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11280933022499084,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.10633862763643265,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10361980646848679,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09461008757352829,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.0526982806622982,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.05010185018181801,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.059819865971803665,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05516386404633522,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.0533674880862236,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.047860048711299896,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04583103954792023,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.030420267954468727,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.026394130662083626,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02526591159403324,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.024998944252729416,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.015230382792651653,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.013057342730462551,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.012946619652211666,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.012082460336387157,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.011911062523722649,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.008006654679775238,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008057527244091034,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007649002596735954,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005406513344496489,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.34.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.09721581637859344,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.09156796336174011,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.08890757709741592,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.0810687243938446,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.04529113322496414,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04288671538233757,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.051874589174985886,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.04798336699604988,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.045895054936409,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.041083645075559616,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.039306074380874634,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.026266034692525864,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.022903643548488617,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02169223316013813,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.02139953151345253,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.013135657645761967,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.011163090355694294,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.011032684706151485,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.010311460122466087,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.01012810692191124,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.006875666324049234,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.006859963294118643,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.006508778315037489,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004488961771130562,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.34.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.22632846236228943,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.21365728974342346,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.20931048691272736,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.1911516934633255,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10600841790437698,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.10133598744869232,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11876922100782394,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.1093919426202774,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10738486796617508,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.096234530210495,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.09192165732383728,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06030182167887688,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.052171818912029266,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05065452679991722,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.05028224736452103,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.030089229345321655,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.025768524035811424,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.025644313544034958,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.0237467959523201,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.023519106209278107,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.015506885945796967,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015166565775871277,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01491775456815958,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.009451237507164478,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.34.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.17889662086963654,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1578661948442459,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.15131935477256775,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.12776286900043488,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.08357130736112595,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.07577972114086151,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.09572537988424301,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.08764807879924774,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.08568286150693893,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.06792788952589035,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.061746951192617416,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.04952565208077431,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.04265158995985985,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.040850862860679626,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.040409330278635025,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.0251143891364336,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.022232549265027046,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.022117530927062035,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.019333481788635254,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.01904735341668129,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.014263826422393322,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015199162997305393,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.013719024136662483,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.011887277476489544,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.34.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.15340903401374817,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.14485864341259003,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.1422334611415863,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.13007429242134094,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.07202481478452682,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.06901099532842636,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.07995815575122833,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.07384327799081802,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.07292751222848892,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.06547898799180984,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.06261275708675385,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.040713291615247726,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.03542444109916687,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.034599121659994125,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.034408099949359894,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.02041930891573429,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.018042193725705147,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.0179844181984663,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.016737444326281548,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.016622494906187057,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.010969391092658043,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.01124835666269064,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.010702168568968773,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.00792392622679472,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.34.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.21682289242744446,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.20492088794708252,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.20126420259475708,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.18405450880527496,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10167765617370605,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.09748056530952454,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11263661831617355,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10414420813322067,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.10291574895381927,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09249300509691238,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.0882653072476387,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.05721987783908844,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.04973486438393593,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.04861532524228096,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04834910109639168,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.028632285073399544,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.024820925667881966,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.02474726364016533,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.022930745035409927,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02276448719203472,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.015033121220767498,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014665515162050724,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.01466311328113079,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009363772347569466,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.34.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.21009665727615356,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.1868971884250641,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.17652013897895813,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.15785232186317444,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.09582196921110153,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.08611485362052917,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.11513795703649521,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.10601633787155151,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.09932798147201538,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.08268258720636368,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.07844188809394836,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.058641236275434494,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.050779931247234344,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.04617611691355705,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.04503439739346504,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.029376281425356865,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.024282891303300858,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.023964539170265198,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.021720917895436287,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.020998887717723846,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.015761541202664375,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.01597749814391136,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.01429897453635931,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.011153185740113258,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.35.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.10566916316747665,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.09954123944044113,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.09697519242763519,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.0884392112493515,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.04923015832901001,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.046775609254837036,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05588714778423309,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.051670461893081665,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.049876753240823746,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.044685639441013336,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.042775318026542664,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.028347987681627274,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.024662157520651817,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02357800304889679,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.023310542106628418,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.014183223247528076,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.012132138013839722,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.012016651220619678,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.01120904553681612,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.01104335580021143,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007429983466863632,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007410021498799324,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007077202666550875,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004860804416239262,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.35.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.09153612703084946,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.08620345592498779,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.08362850546836853,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.07621956616640091,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.04258275032043457,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04028512164950371,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.04885905236005783,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.04517258331179619,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.04314275458455086,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.03862963989377022,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.03696572035551071,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.02474045939743519,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02157101035118103,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.020391009747982025,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.020107414573431015,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.012372422032058239,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.010490021668374538,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.01036033034324646,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.009691270999610424,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.009515905752778053,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.006477865390479565,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.006452817004173994,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.006119238678365946,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004216037690639496,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.35.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.2232120782136917,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.2105170041322708,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.20611825585365295,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.18815767765045166,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10426394641399384,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.09961657971143723,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11702008545398712,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10780958831310272,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10558590292930603,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.09465775638818741,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.09045783430337906,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.05938558280467987,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05141317844390869,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.0497785247862339,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.049405090510845184,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.029630254954099655,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02533992938697338,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.025198545306921005,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.02336709387600422,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.023123441264033318,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.015293481759727001,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.014957794919610023,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01468225009739399,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.009331458248198032,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.35.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.1958778351545334,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.17050758004188538,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.16148751974105835,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.13066405057907104,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.09059976786375046,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.08108698576688766,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.10676587373018265,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.09792418777942657,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.09404212981462479,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07240436971187592,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.064571313560009,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.05530649051070213,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.04768076539039612,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04434400796890259,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.043512776494026184,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.028084615245461464,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.024235164746642113,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.023989828303456306,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.020879773423075676,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.020372148603200912,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.015975255519151688,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01687926985323429,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01494332030415535,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.013156832195818424,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.35.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.15588068962097168,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.14725829660892487,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.144595205783844,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.13229380548000336,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.07313189655542374,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07009433954954147,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.08113972842693329,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.07499495893716812,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.07404889166355133,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.06653086841106415,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.06362920254468918,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.04132406413555145,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.03594429790973663,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.035101473331451416,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.03490975871682167,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.0207064189016819,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.01824997365474701,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.018189819529652596,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.0169325303286314,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.01681109517812729,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.011070693843066692,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.011294934898614883,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.010801956057548523,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.00786822009831667,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.35.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.2159193605184555,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.20419193804264069,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.20053626596927643,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.18351136147975922,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10114000737667084,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.09699691087007523,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11201728880405426,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10359305888414383,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.10237665474414825,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.092049241065979,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.08801556378602982,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.056847114115953445,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.04946814849972725,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.04835369065403938,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04808779060840607,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.02841763012111187,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.024710914120078087,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.02463081292808056,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02284422144293785,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.022673314437270164,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.0148174362257123,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014632830396294594,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.014446938410401344,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009391602128744125,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.35.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.22096911072731018,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.19622331857681274,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.18586042523384094,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.16627196967601776,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.10078048706054688,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.09075574576854706,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.12097712606191635,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11052947491407394,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.10452351719141006,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.08684183657169342,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08241590857505798,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06153082102537155,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.053042978048324585,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.04862840846180916,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.04754592478275299,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.030961500480771065,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.025703810155391693,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.025427931919693947,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.02301599830389023,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.022329721599817276,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.01687803491950035,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.017004407942295074,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.015458458103239536,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.012114658020436764,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.36.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.1051112487912178,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.09896844625473022,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.09647226333618164,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.08794190734624863,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.04895959421992302,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.046552930027246475,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.055677104741334915,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05128919705748558,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.0496022030711174,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.044442884624004364,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04256904497742653,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.02827363833785057,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.024503180757164955,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.023446986451745033,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.023191429674625397,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.014151547104120255,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.012067319825291634,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.011963502503931522,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.01115417666733265,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.010994003154337406,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007431637495756149,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007376988418400288,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007092849817126989,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004862716421484947,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.36.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.09036833792924881,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.08505772054195404,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.08241179585456848,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.07506591081619263,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.04197003319859505,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.039652612060308456,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.0484071746468544,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.04474818706512451,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.042528219521045685,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.03807076811790466,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.03643353655934334,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.024508027359843254,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.021342875435948372,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.020113812759518623,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.01981891691684723,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.012269609607756138,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.010382739827036858,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.010245323181152344,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.009602210484445095,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.009415369480848312,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.00646283570677042,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.0064459433779120445,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.006085617002099752,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.0042722830548882484,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.36.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.22861766815185547,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.215643048286438,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.21146930754184723,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.1929529756307602,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.1068945825099945,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.10216604173183441,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11958026885986328,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11014476418495178,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10822433978319168,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.09702764451503754,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.09267585724592209,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06071590632200241,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05252303183078766,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05100943148136139,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.05064873769879341,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03027883544564247,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.025916745886206627,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02579529769718647,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.023879345506429672,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02365603670477867,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01559043675661087,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015191800892353058,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.015000914223492146,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.009378504008054733,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.36.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.188729926943779,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.16385392844676971,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.1554960012435913,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.1283813714981079,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.08671551942825317,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.07790794223546982,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.10194519907236099,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.09310822188854218,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.0897015929222107,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.06965379416942596,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.06258305162191391,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.05243917182087898,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.04551834613084793,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04267016425728798,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04198800399899483,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.026663726195693016,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02365526184439659,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.023446671664714813,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.02057207003235817,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02013891190290451,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.015335996635258198,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.016771920025348663,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.014514036476612091,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.013433151878416538,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.36.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.15447470545768738,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.14591702818870544,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.1433015763759613,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.13115398585796356,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.07246635109186172,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.0694664940237999,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.08024752885103226,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.07425501197576523,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.073348268866539,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.06590353697538376,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.06293154507875443,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.04087391495704651,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.035540904849767685,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.0347200408577919,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.03453494980931282,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.02042570896446705,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.01797688566148281,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.01792219467461109,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.01666218973696232,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.016544917598366737,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.010797007940709591,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.011006258428096771,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.01053224690258503,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.0075215003453195095,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.36.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.21227394044399261,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.20070108771324158,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.19713860750198364,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1803971827030182,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.09945371747016907,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.0953909382224083,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.10997548699378967,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10182436555624008,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.10066014528274536,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09051205217838287,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.0863790363073349,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.055831488221883774,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.04856890067458153,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.04750147834420204,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04724648594856262,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.027855725958943367,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02416413649916649,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.02409224957227707,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.022325320169329643,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02216184511780739,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.014421277679502964,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014160383492708206,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.014057448133826256,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.008871468715369701,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.36.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.21349436044692993,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.1892443150281906,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.17873834073543549,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.15992262959480286,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.09718415886163712,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.08724085241556168,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.11712458729743958,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.10747825354337692,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.10086696594953537,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.08371268212795258,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.07955390214920044,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.05967243015766144,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05153829976916313,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.046920422464609146,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.04578296095132828,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03001829795539379,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.024838311597704887,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.024530675262212753,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.022234391421079636,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.021515296772122383,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.01637515425682068,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.01652871072292328,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.014907002449035645,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.01178024709224701,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.37.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.1089792549610138,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.10281878709793091,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10037878900766373,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09165811538696289,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.050829723477363586,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04843325540423393,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05729413032531738,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.053017955273389816,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.051504939794540405,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.046189118176698685,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04412824288010597,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.029045918956398964,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02528560906648636,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.024306653067469597,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.024073543027043343,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.014515100978314877,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.012431672774255276,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.012337033636868,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.011479285545647144,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.011333441361784935,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007566496729850769,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007462440058588982,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.0072458176873624325,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004768254701048136,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.37.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.09665566682815552,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.09113030880689621,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.0887870118021965,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.08107111603021622,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.0450587272644043,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04282006621360779,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05117926374077797,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.04726549610495567,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.0456300713121891,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04093145579099655,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.039205946028232574,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.025918414816260338,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.022529780864715576,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.021544352173805237,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.021304745227098465,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.012951918877661228,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.011022952385246754,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.010918889194726944,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.010183367878198624,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.010030454955995083,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.006736645940691233,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.0066476259380578995,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.00641799159348011,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004245365969836712,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.37.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.23068790137767792,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.21784546971321106,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.2136693298816681,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.1953597068786621,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10797702521085739,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.10336530953645706,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12034261226654053,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11104518920183182,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.109344981610775,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.09818338602781296,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.09379004687070847,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06108224019408226,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.052944716066122055,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05156582593917847,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.05121917277574539,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.030457042157649994,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.026190591976046562,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.026085134595632553,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.024170715361833572,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02395772933959961,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.015690919011831284,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015320773236453533,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.015169764868915081,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.009477815590798855,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.37.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.21077893674373627,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.16832713782787323,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.15208624303340912,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.12484442442655563,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.09522708505392075,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.07862722128629684,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11690939217805862,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10727312415838242,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10111062228679657,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.06997895985841751,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.06472863256931305,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06024964153766632,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05188923329114914,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.046431802213191986,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04505963250994682,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.030337728559970856,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.025109002366662025,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.024718299508094788,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.020438406616449356,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.019512537866830826,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01682381145656109,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01747906021773815,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.015134865418076515,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.013077395968139172,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.37.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.1651444137096405,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.15604540705680847,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.15320689976215363,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1401614546775818,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.07746388018131256,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07424996048212051,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.08576510101556778,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.07938850671052933,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.07842721790075302,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.0704340785741806,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.06728894263505936,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.04364791885018349,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.03798624873161316,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.03712788596749306,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.03692885488271713,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.02180868573486805,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.019200285896658897,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.019144631922245026,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.01779891736805439,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.01767386682331562,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.011500717140734196,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.011734472587704659,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.011220438405871391,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.008006780408322811,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.37.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.22064347565174103,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.20863860845565796,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.20498250424861908,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.18758636713027954,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10336142778396606,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.0991603210568428,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11451541632413864,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10578961670398712,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.1046295240521431,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09407995641231537,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.08994792401790619,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.05818326398730278,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05050040781497955,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.04939163848757744,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04913296923041344,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.02909340336918831,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.025202587246894836,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.025121942162513733,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.023290708661079407,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02313142828643322,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.015239952132105827,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014864549972116947,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.014876571483910084,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.00945835467427969,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.37.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.21733501553535461,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.19295631349086761,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.18202562630176544,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.16287653148174286,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.09898295253515244,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.08878424763679504,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.11962822079658508,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.1098419576883316,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.10277144610881805,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.0854543000459671,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08114872872829437,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.060682512819767,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.052735473960638046,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.04783505201339722,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.04661988466978073,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03045915998518467,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.025399021804332733,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.025051558390259743,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.02280254103243351,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.022033384069800377,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.016405006870627403,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.01702183112502098,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.014854107052087784,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.012197521515190601,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.38.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.10840373486280441,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.10203465074300766,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.09951820969581604,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.0906880721449852,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05050083249807358,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04800054430961609,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05708182975649834,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.052801910787820816,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05115027725696564,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.045788709074258804,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04369981586933136,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.02893211506307125,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.025207355618476868,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02414996549487114,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.023899052292108536,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.014461449347436428,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.012379087507724762,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.01227356493473053,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.01142218615859747,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.01126299798488617,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007539711892604828,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007483058609068394,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007197677623480558,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.0048324940726161,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.38.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.09354235231876373,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.08808208256959915,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.08561381697654724,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.07805363833904266,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.043549347668886185,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04126729443669319,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.049832895398139954,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.046012215316295624,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.044117897748947144,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.039503421634435654,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.03779906406998634,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.02524418570101261,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02195681817829609,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02084388956427574,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.020577529445290565,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.01262418832629919,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.01069556549191475,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.010578296147286892,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.009877169504761696,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.009711123071610928,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.006597273517400026,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.006520950235426426,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.006247222423553467,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004207333084195852,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.38.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.23128223419189453,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.21821317076683044,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.21389484405517578,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.1952914446592331,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.1082669049501419,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.1035652831196785,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12085577100515366,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11146052181720734,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10965362191200256,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.09826874732971191,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.09381882846355438,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06135355308651924,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05313543230295181,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.051695313304662704,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.051345475018024445,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.030582385137677193,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02625182643532753,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.026139112189412117,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.02418607287108898,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.023965587839484215,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.015741076320409775,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015359469689428806,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.015185849741101265,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.009470593184232712,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.38.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.19739507138729095,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1680898666381836,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.15899738669395447,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.13582609593868256,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.09092827886343002,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.07981686294078827,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.10507354885339737,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.09671581536531448,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.09444867074489594,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07297962158918381,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.06499844044446945,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.05407273769378662,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.046763621270656586,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.044172778725624084,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04353917017579079,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.027211153879761696,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.023630935698747635,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.023488765582442284,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.020224912092089653,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.019809696823358536,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.014947950839996338,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015785925090312958,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01413938868790865,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.011810810305178165,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.38.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.17412762343883514,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.1645086407661438,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.16155356168746948,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.14774960279464722,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.08168185502290726,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07832084596157074,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.09041059017181396,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.0836837887763977,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.08270034193992615,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.07432214170694351,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.07094278186559677,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.04602807015180588,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.04004215821623802,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.039146970957517624,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.038941387087106705,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.022989317774772644,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02022891491651535,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.020167512819170952,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.018741805106401443,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.01860964111983776,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.012124987319111824,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.01232968084514141,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.011830165982246399,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.00836977083235979,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.38.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.22457760572433472,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.21233196556568146,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.2085517793893814,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.19078150391578674,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10524765402078629,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10096214711666107,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11638610064983368,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10775063186883926,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.1065412238240242,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09579352289438248,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09140048176050186,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.05911499261856079,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05141357332468033,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.0502750463783741,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.050013218075037,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.029489664360880852,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.025589335709810257,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.025514639914035797,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.023629970848560333,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.023464087396860123,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.01528619322925806,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014997215010225773,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.014914660714566708,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009402723982930183,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.38.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.21930202841758728,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.19469766318798065,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.18397413194179535,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.16450870037078857,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.09979492425918579,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.08978252112865448,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.12031866610050201,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11040101945400238,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.10369422286748886,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.0863056555390358,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08174511045217514,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06118005886673927,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05298508703708649,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.04818781837821007,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.04700232297182083,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.030994737520813942,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.025507111102342606,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.02518223039805889,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.02290397509932518,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.022154923528432846,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.017225945368409157,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.016968898475170135,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.01578715816140175,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.012071490287780762,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.39.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11096680164337158,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.10444629192352295,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10183646529912949,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09288892894983292,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05172320082783699,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04916127771139145,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05875239148736,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05417440086603165,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05243905261158943,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.046942390501499176,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04490981996059418,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.029795151203870773,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02587123215198517,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.024777108803391457,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.024504823610186577,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.01493213139474392,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.012712176889181137,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.012604213319718838,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.011738218367099762,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.011573528870940208,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007852237671613693,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007717600092291832,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007493746466934681,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.00501406192779541,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.39.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.09763119369745255,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.0919230580329895,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.08947167545557022,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.08159884065389633,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.04551579803228378,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.043187253177165985,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05184590071439743,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.047874290496110916,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.04613787308335304,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04128556326031685,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.03948301821947098,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.026271376758813858,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02284562960267067,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02177165262401104,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.021518900990486145,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.013132225722074509,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.011142241768538952,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.0110334362834692,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.010281256400048733,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.010120006278157234,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.006835321895778179,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.006729904096573591,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.0064919437281787395,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004289011005312204,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.39.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.2316640168428421,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.21860936284065247,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.21425507962703705,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.1957615315914154,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10855774581432343,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.10383632034063339,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12111705541610718,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11173340678215027,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10992462933063507,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.09852556884288788,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.0940122902393341,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.061490826308727264,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.053280074149370193,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05182000994682312,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.05147331580519676,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.030669301748275757,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.026318373158574104,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02620471827685833,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.024244440719485283,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.024021023884415627,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.015781208872795105,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015398204326629639,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.015218718908727169,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.00948776863515377,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.39.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.15466228127479553,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1300457864999771,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.12327944487333298,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09872683882713318,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.0699872076511383,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.06184358149766922,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.08166354894638062,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.07431445270776749,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.07243284583091736,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.05464212968945503,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04837505519390106,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.041938211768865585,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.03669803962111473,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.03472238406538963,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.03425436094403267,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.02168845757842064,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.01972053572535515,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.01961800456047058,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.017064178362488747,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.016773415729403496,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.012855703011155128,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01440359465777874,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.012332193553447723,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.011979450471699238,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.39.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.18267588317394257,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.1725618690252304,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.16942554712295532,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1549225151538849,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.0857386365532875,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.08217731863260269,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.09491550922393799,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.08783674240112305,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.08680743724107742,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.07793749868869781,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.07437630742788315,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.04834009334445,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.042019546031951904,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.0410698838531971,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04084636643528938,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.024156004190444946,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.0211617611348629,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.021099446341395378,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.01958865486085415,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.019444642588496208,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.0127252247184515,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.012810461223125458,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.012415547855198383,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.008580388501286507,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.39.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.22977252304553986,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.21713458001613617,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.21332760155200958,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1950647532939911,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10778293758630753,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10337267816066742,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11930335313081741,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.11035008728504181,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.10909917205572128,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09802955389022827,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09362795203924179,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06071979179978371,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05269822105765343,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.05153369531035423,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05125818029046059,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.030362049117684364,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.026312818750739098,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.02623649686574936,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.024308273568749428,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.024135854095220566,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.01589910499751568,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.015551671385765076,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.015517661347985268,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009952361695468426,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.39.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.22570346295833588,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.20238251984119415,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.19292014837265015,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.17284977436065674,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.10298380255699158,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.09379874914884567,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.12144582718610764,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11200577765703201,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.1064911037683487,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.08969957381486893,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08486056327819824,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06173224747180939,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05355139076709747,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.04947085306048393,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.04847092181444168,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.030963648110628128,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.025707073509693146,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.025462860241532326,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.02307041361927986,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.022430777549743652,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.016548769548535347,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.016371427103877068,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.015250291675329208,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.011026235297322273,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.40.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11246956139802933,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.10589724779129028,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10321171581745148,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09417930245399475,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05248071253299713,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.049888499081134796,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.059368252754211426,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05486034229397774,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05316570773720741,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.047564536333084106,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04546622186899185,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.03011399134993553,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.026181822642683983,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.025108100846409798,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.02484753727912903,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.015048550441861153,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.012864328920841217,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.012756875716149807,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.011868763715028763,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.011705847457051277,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.00786946527659893,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007770999800413847,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.00752690015360713,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005009807646274567,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.40.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.09824681282043457,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.09253616631031036,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.08999691903591156,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.08210963755846024,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.045867323875427246,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04348492622375488,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05230575427412987,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.0482807494699955,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.04646465927362442,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04160727187991142,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.03980463370680809,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.026546671986579895,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02304394543170929,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.021943550556898117,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.02168048731982708,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.013262372463941574,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.011231016367673874,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.011115308851003647,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.010356679558753967,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.010194359347224236,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.006914212368428707,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.006786372046917677,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.006569885648787022,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004326221998780966,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.40.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.23705460131168365,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.2236633002758026,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.2193603217601776,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.2003420889377594,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.11111904680728912,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.10624542087316513,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12390764057636261,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11431929469108582,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.1125192940235138,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.10083930939435959,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.09625069797039032,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.0628923699259758,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05451743304729462,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.053042925894260406,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.052695974707603455,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03136008232831955,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.026924453675746918,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.026805561035871506,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.02480492740869522,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.024576054885983467,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01610778458416462,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015723001211881638,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.015543656423687935,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.009670289233326912,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.40.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.19925038516521454,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.17183630168437958,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.16098074615001678,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.1343812197446823,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.0923461988568306,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.08083637058734894,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11002194881439209,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10124272853136063,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.09516192972660065,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07219384610652924,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.06723490357398987,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.05663392320275307,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.04877771809697151,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04470192641019821,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04369831830263138,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.028441613540053368,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.023664169013500214,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.023276634514331818,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.019767148420214653,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.01907832734286785,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.015466130338609219,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015706036239862442,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.014191271737217903,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.011228177696466446,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.40.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.18798178434371948,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.17753183841705322,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.17433665692806244,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.15943217277526855,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.08825644105672836,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.08461453765630722,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.09781307727098465,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.09040916711091995,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.08937369287014008,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.08022439479827881,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.07661228626966476,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.04976256191730499,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.0432460717856884,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.0422748401761055,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04204583168029785,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.024881752207875252,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02179538644850254,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.02172798104584217,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02016936056315899,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02002745307981968,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.013122627511620522,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.013214055448770523,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.012804390862584114,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.008880531415343285,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.40.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.2315833866596222,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.2188558429479599,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.2149510681629181,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.19653059542179108,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10862010717391968,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10414640605449677,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.1202351450920105,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.1111917495727539,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.10995333641767502,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09874442219734192,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09426873922348022,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06107863411307335,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05307835340499878,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.05190717428922653,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05162108689546585,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.030514134094119072,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.026475006714463234,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.026390742510557175,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.024445828050374985,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.024265186861157417,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.01591346226632595,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.015594523400068283,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.015533271245658398,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009898025542497635,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.40.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.2293366938829422,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.20572873950004578,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.1961810141801834,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.17558914422988892,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.10464917868375778,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.09541332721710205,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.12335626035928726,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11373503506183624,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.1081886887550354,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09107358008623123,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08606656640768051,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06252028048038483,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.054262131452560425,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.050192032009363174,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.04919518530368805,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.031240640208125114,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.02593991532921791,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.025689153000712395,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.02322237752377987,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.02257816679775715,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.016439180821180344,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.016320709139108658,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.015116886235773563,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.010745406150817871,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.41.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11075275391340256,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1042572483420372,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.1015777513384819,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09270202368497849,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.051681846380233765,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04909965395927429,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05877452716231346,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05414637550711632,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05237541347742081,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.0468650683760643,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04486500844359398,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.029809651896357536,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.025847870856523514,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.024731485173106194,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.024465706199407578,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.014906332828104496,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.012669564224779606,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.012559864670038223,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.011685891076922417,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.011517385020852089,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.0077865226194262505,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007646826095879078,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007425371091812849,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.00490790419280529,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.41.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.098542720079422,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.09275997430086136,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.09022080898284912,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.08235343545675278,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.04590918868780136,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04355127736926079,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.052689094096422195,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.0484122596681118,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.04655982181429863,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04164896532893181,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.03998396545648575,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.02667500264942646,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.023103274405002594,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.021988801658153534,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.021719256415963173,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.01334065105766058,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.011264655739068985,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.011146578006446362,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.010392450727522373,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.010224323719739914,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.006973589304834604,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.006821438204497099,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.0066180964931845665,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004362835083156824,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.41.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.2374633103609085,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.2240619957447052,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.21969564259052277,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.20062333345413208,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.1114245131611824,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.1065237820148468,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12433436512947083,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11461872607469559,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.11284070461988449,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.10107889026403427,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.09652155637741089,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06318865716457367,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05471699684858322,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05323363468050957,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.052880898118019104,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03152924403548241,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.027111435309052467,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.026996664702892303,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.024975212290883064,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.024756740778684616,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.016279589384794235,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01596667803823948,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.015709348022937775,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.010021365247666836,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.41.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.2019592970609665,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.16927897930145264,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.15719377994537354,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.13315950334072113,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.09186895936727524,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.07878749072551727,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11123888939619064,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10163969546556473,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.09655670076608658,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07159284502267838,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.06577030569314957,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.057249780744314194,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.04955126345157623,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04514329135417938,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04405990242958069,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.029001932591199875,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.024935193359851837,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.024629751220345497,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.02126009203493595,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02058251015841961,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.016439110040664673,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01776205003261566,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.015147740021348,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.01397156435996294,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.41.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.19191880524158478,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.18121397495269775,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.17792096734046936,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.16269895434379578,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.09013035893440247,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.08637895435094833,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.09978921711444855,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.09233997017145157,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.09125803411006927,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.08189457654953003,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.07812829315662384,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.05079242214560509,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.04415352642536163,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.043158240616321564,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04292380437254906,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.025375008583068848,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.022239213809370995,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.022175002843141556,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02057591639459133,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02042802795767784,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.013354343362152576,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.013465909287333488,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.013029159978032112,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009031702764332294,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.41.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.23301953077316284,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.22017115354537964,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.2162368893623352,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.19768869876861572,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10928916186094284,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10476808995008469,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.12095765769481659,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.11192703247070312,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.11065579205751419,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09934201091527939,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.0948738306760788,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06143806129693985,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05341527983546257,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.05222969874739647,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05194483697414398,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.030688779428601265,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02659681811928749,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.02651509828865528,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.0245366208255291,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02435891143977642,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.015956759452819824,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.015607054345309734,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.01556381769478321,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009817877784371376,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.41.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.22941261529922485,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.20638422667980194,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.19690300524234772,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.1761244237422943,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.10494203120470047,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.09578920155763626,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.12359963357448578,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11406944692134857,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.10837250202894211,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09154333919286728,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.0864722952246666,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06292180716991425,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05463135614991188,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.05047821253538132,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.04945354163646698,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03166795149445534,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.026343535631895065,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.02607893757522106,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.023698443546891212,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.023053869605064392,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.01694663055241108,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.016919896006584167,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.015585961751639843,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.01158936507999897,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.42.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.1207377016544342,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.11368094384670258,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.11097180843353271,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.10123412311077118,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05641186237335205,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.05368398502469063,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.06361094117164612,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05884888768196106,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05716293677687645,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.051120925694704056,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04879286140203476,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.03223768621683121,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.028063606470823288,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.026976309716701508,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.02671450562775135,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.016089359298348427,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.013768477365374565,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.013663900084793568,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.012683689594268799,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.012517094612121582,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.008352350443601608,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008225636556744576,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.00799147691577673,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005197098944336176,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.42.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.10598204284906387,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.09979574382305145,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.09725097566843033,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.0887870043516159,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.04955014958977699,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04711258411407471,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05623515695333481,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05188717320561409,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.050210095942020416,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04494965448975563,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.043093834072351456,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.028481096029281616,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.024768562987446785,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02371167577803135,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.0234625693410635,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.014238893985748291,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.012123639695346355,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.012020091526210308,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.011184850707650185,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.011026942171156406,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007402410265058279,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007290550507605076,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.00706443190574646,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004646646790206432,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.42.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.24030716717243195,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.22654861211776733,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.22225125133991241,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.2028857171535492,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.11269000917673111,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.10776543617248535,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12548142671585083,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11581489443778992,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.11412506550550461,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.10220611840486526,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.09751400351524353,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06375699490308762,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05525459721684456,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05382045358419418,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.053482785820961,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03179574012756348,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02734253741800785,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02723177708685398,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.025164643302559853,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.024949193000793457,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.016366643831133842,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015981296077370644,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.015820052474737167,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.009885377250611782,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.42.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.20028363168239594,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.17244982719421387,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.1624222695827484,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.1382751613855362,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.09208753705024719,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.08112458884716034,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.1082477793097496,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.09978880733251572,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.09541626274585724,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07432852685451508,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.06763827055692673,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.05558134987950325,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.047948047518730164,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04436210170388222,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04348943382501602,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.0278971865773201,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.023193182423710823,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.022898532450199127,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.01968897134065628,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.01909307949244976,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.014940506778657436,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.014936505816876888,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.013767009600996971,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.010287345387041569,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.42.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.19681492447853088,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.18583229184150696,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.18246358633041382,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.16674622893333435,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.09246011823415756,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.08859779685735703,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.10239467769861221,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.0947360098361969,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.09363264590501785,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.08397845923900604,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.08013687282800674,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.05218895152211189,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.045309171080589294,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.0442732498049736,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.0440310463309288,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.026089206337928772,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.022779477760195732,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.022716525942087173,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02106141857802868,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.0209110826253891,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.01377092208713293,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.013745625503361225,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.013439816422760487,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009160437621176243,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.42.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.2351755052804947,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.22216427326202393,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.21812854707241058,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.19940033555030823,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.110328808426857,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.1057349219918251,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.12204652279615402,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.11300735920667648,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.1117284893989563,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.10023315995931625,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09560829401016235,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06203010305762291,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05391262099146843,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.05270540341734886,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05242065340280533,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.030951213091611862,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.026804648339748383,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.026724644005298615,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.024718286469578743,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02453945204615593,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.016064513474702835,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.015684721991419792,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.01566636562347412,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009797458536922932,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.42.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.23384714126586914,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.210712268948555,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.20141291618347168,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.1798439770936966,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.10688336938619614,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.09787844121456146,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.12556329369544983,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11572398990392685,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.1102985069155693,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09323344379663467,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08789011836051941,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06357724219560623,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.055168602615594864,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.0512029230594635,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.05022841691970825,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03182746469974518,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.02632906660437584,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.026083094999194145,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.023575305938720703,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.02294759824872017,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.016773967072367668,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.016356224194169044,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.015474039129912853,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.010532588697969913,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.43.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11975536495447159,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.11271099001169205,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.109856516122818,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.10014555603265762,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05597492307424545,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.05313858762383461,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.06339426338672638,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05860183760523796,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.056722745299339294,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.05067373067140579,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.0483873225748539,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.03218298777937889,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.027981964871287346,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.026787079870700836,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.026498623192310333,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.0160963237285614,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.013732061721384525,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.013611366041004658,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.012652353383600712,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.012469598092138767,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.008420761674642563,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008305119350552559,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.008031852543354034,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005355187691748142,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.43.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.10474711656570435,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.09857699275016785,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.09598613530397415,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.08750969171524048,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.04895006865262985,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04642871767282486,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05586894974112511,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05148501321673393,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.04960639402270317,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.044369302690029144,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04246442764997482,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.02831682562828064,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02456892654299736,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.0234312042593956,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.023165157064795494,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.014157992787659168,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.011989983730018139,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.011870965361595154,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.011045234277844429,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.010876917280256748,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.0073767853900790215,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007231999654322863,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007013323251157999,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004606429487466812,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.43.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.24012847244739532,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.22622285783290863,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.22177539765834808,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.20227694511413574,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.11250581592321396,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.10747182369232178,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.1255837082862854,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11583754420280457,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.11398380249738693,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.101911760866642,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.0972607284784317,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06373625248670578,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05523320287466049,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05370320752263069,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.05333370342850685,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03178776428103447,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02724919468164444,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.027123818174004555,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.025056960061192513,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.024825366213917732,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.016321755945682526,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015898441895842552,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01573893614113331,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.00974219385534525,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.43.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.2017962783575058,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1515500247478485,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.1348421573638916,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.11322059482336044,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.0866287350654602,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.06949738413095474,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11033110320568085,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.09943799674510956,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.0932995155453682,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.06454340368509293,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.06058870255947113,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.0550636351108551,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.049790676683187485,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.044053561985492706,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04255734756588936,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.02858087420463562,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.026483599096536636,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02614731714129448,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.022968288511037827,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.022170288488268852,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.016947254538536072,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.020996050909161568,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.015382563695311546,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.018074458464980125,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.43.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.2013033926486969,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.19000519812107086,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.18649113178253174,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.17035844922065735,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.09461657702922821,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.09061675518751144,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.10486403852701187,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.09695155918598175,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.09584123641252518,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.08586309105157852,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.08187933266162872,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.05338219180703163,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.046394504606723785,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.04533606022596359,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.045082539319992065,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.02675141580402851,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02336738631129265,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.023297104984521866,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.021598655730485916,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02144249528646469,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.014206402003765106,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014147168025374413,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.013866924680769444,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.00948359351605177,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.43.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.2379877269268036,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.22471685707569122,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.22062602639198303,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.2016104906797409,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.11172069609165192,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10702896118164062,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.1238275095820427,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.11444728821516037,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.1131439208984375,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.10141754895448685,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09676030278205872,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06288962066173553,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.054667308926582336,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.05343201383948326,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.0531412735581398,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.031501542776823044,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.027281688526272774,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.027200695127248764,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02517687901854515,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02498367615044117,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.01658128947019577,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.016123659908771515,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.016176704317331314,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.010293086059391499,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.43.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.2333824187517166,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.21077537536621094,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.20126008987426758,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.17987723648548126,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.10694903880357742,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.09788631647825241,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.12587501108646393,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11625368148088455,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.11028823256492615,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09352607280015945,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08827385306358337,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06393296271562576,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05562245845794678,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.05138111859560013,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.050349924713373184,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.032057344913482666,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.02671252377331257,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.02642405778169632,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.024043457582592964,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.023379355669021606,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.016939735040068626,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.017010897397994995,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.015538839623332024,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.011454755440354347,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.44.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.12429948151111603,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.11701738834381104,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.11424943059682846,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.1041572242975235,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05816012993454933,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.05532520264387131,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.06551264971494675,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.060608915984630585,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.058932024985551834,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.05266536772251129,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.050192348659038544,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.03318256139755249,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02891453728079796,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.027794605121016502,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.02753586694598198,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.016568733379244804,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.014174566604197025,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.014067666605114937,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.013039682060480118,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.01287721935659647,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.008578050881624222,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008431239984929562,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.008213955909013748,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005283293314278126,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.44.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.1104007288813591,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.10394234210252762,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10136254131793976,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09240496158599854,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05165312439203262,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04907660186290741,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05855358764529228,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.0540103018283844,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05230960249900818,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04674394428730011,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.0446731299161911,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.029696250334382057,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02576027810573578,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02470601536333561,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.024452906101942062,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.014839941635727882,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.012616329826414585,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.012510277330875397,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.011612208560109138,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.011454218067228794,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007707976270467043,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007550135254859924,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.00736978417262435,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.0047690593637526035,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.44.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.24462206661701202,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.23065614700317383,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.22613155841827393,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.2064206898212433,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.11475969851016998,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.10971886664628983,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12788961827754974,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.1180306077003479,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.11627932637929916,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.10398813337087631,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.0992286205291748,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06492634862661362,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05630563572049141,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05481645092368126,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.054469939321279526,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.032395053654909134,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.027799755334854126,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.027688179165124893,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.025559386238455772,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02533893845975399,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.0166392270475626,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.016190461814403534,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.0160885788500309,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.009909584186971188,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.44.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.21443980932235718,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.17065373063087463,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.15592512488365173,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.13832297921180725,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.09708372503519058,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.07850441336631775,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11565673351287842,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10631882399320602,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10254886001348495,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07325955480337143,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.06981109827756882,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.05948180705308914,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05119756981730461,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04698676988482475,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.045950088649988174,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.02989501878619194,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.024749401956796646,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.024500513449311256,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.020084695890545845,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.019347824156284332,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.016282539814710617,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.016242899000644684,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.014964313246309757,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.011433583684265614,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.44.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.2056315541267395,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.1940189152956009,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.1904650777578354,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.17405687272548676,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.09668603539466858,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.09258531033992767,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.10728713870048523,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.09909595549106598,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.09792713820934296,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.0877152532339096,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.0837249755859375,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.05466708168387413,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.04741859436035156,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.04632400721311569,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.046071857213974,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.02735697105526924,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02388397976756096,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.02380960062146187,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02206692099571228,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.021905258297920227,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.014500955119729042,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014472313225269318,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.01415200810879469,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009707675315439701,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.44.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.24058176577091217,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.22702756524085999,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.22286681830883026,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.20367014408111572,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.11294931918382645,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10816209018230438,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.12497486174106598,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.11571726948022842,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.11439248919487,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.10249335318803787,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09768789261579514,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06354234367609024,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05523085221648216,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.053979549556970596,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.053682561963796616,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.03170476108789444,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.027461810037493706,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.027376942336559296,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.025293132290244102,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.025102738291025162,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.016440104693174362,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.016075605526566505,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.016026023775339127,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.010042441077530384,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.44.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.23909775912761688,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.21587511897087097,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.2063360959291458,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.18412482738494873,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.10954999923706055,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.1003900021314621,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.1290895789861679,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11881605535745621,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.1129547581076622,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09569262713193893,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.09023589640855789,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06566064059734344,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05674010515213013,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.05257747694849968,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.05155808478593826,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03308631107211113,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.02720097079873085,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.026927588507533073,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.0244283489882946,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.023771069943904877,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.017835039645433426,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.017131546512246132,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.016477731987833977,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.011325626634061337,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.45.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.1292351335287094,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.12166624516248703,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.11880974471569061,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.10822224617004395,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.06046082824468613,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.05752069130539894,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.06825235486030579,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.06301330029964447,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.06127508357167244,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.054783742874860764,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.052186135202646255,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.0346309132874012,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.030076870694756508,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.028928633779287338,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.028654422610998154,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.017309442162513733,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.014776871539652348,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.014672348275780678,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.01360686868429184,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.013431871309876442,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.009016803465783596,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008842778392136097,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.008646970614790916,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005619286093860865,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.45.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.1138591468334198,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1072608232498169,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10454373806715012,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09534899145364761,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.053367048501968384,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.050688836723566055,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.060616765171289444,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.055813997983932495,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05406571924686432,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04832778871059418,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.046199213713407516,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.03077462874352932,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.026656273752450943,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02553272806107998,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.025266624987125397,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.015379545278847218,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.013054153881967068,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.012943459674715996,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.012022621929645538,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.01185524370521307,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.008008044213056564,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007836073637008667,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007647458929568529,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004975686781108379,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.45.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.2461414933204651,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.23201829195022583,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.22743700444698334,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.2073979377746582,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.11551246047019958,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.11037097871303558,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12873238325119019,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11874964088201523,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.11702542752027512,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.10457030683755875,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.09973790496587753,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06530828028917313,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05664623901247978,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.0551680363714695,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.05482243001461029,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.032577719539403915,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02800152450799942,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.027892261743545532,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.025730205699801445,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.025505363941192627,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01675841212272644,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.016339031979441643,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.016199396923184395,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.010050706565380096,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.45.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.2239958941936493,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.18837031722068787,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.17617060244083405,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.14451907575130463,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10122643411159515,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.08944576233625412,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.1208551675081253,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11130166053771973,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10705909878015518,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07838417589664459,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.07251095026731491,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06201254948973656,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05351325869560242,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.0488952174782753,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04776318371295929,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03103882446885109,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02570759691298008,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02543891780078411,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.021278465166687965,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.020483965054154396,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.016667313873767853,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01687668263912201,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.015182160772383213,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.0117912907153368,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.45.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.2031402587890625,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.19160358607769012,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.18801981210708618,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.17166955769062042,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.09556519985198975,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.0914776399731636,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.10592743754386902,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.09797966480255127,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.09679757058620453,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.08664871007204056,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.08258221298456192,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.053998492658138275,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.04688424989581108,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.045791033655405045,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.045531757175922394,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.026984548196196556,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.023585885763168335,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.02351224422454834,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.021767508238554,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02160777896642685,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.014234478585422039,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014266119338572025,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.013876987621188164,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009527021087706089,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.45.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.24147464334964752,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.22782766819000244,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.2236160784959793,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.20414739847183228,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.11347118020057678,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10861805826425552,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.12576335668563843,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.11630499362945557,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.11493213474750519,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.10287962108850479,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.0980646163225174,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06397948414087296,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05556006729602814,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.05426651984453201,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05396265909075737,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.03193706274032593,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.027700480073690414,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.027615325525403023,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02551579661667347,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.025325629860162735,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.016661321744322777,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.016363009810447693,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.016232987865805626,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.010415230877697468,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.45.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.24244774878025055,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.2189178764820099,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.20906463265419006,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.18669144809246063,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.1111607700586319,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.10180613398551941,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.13055607676506042,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.12076055258512497,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.11459416896104813,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09711325913667679,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.09149108827114105,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06630448251962662,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05771786719560623,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.05340854451060295,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.05236430838704109,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03322070464491844,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.027765929698944092,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.027478644624352455,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.02497418224811554,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.024300657212734222,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.017600174993276596,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.01766425371170044,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.01622716709971428,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.011908263899385929,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.46.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.12590506672859192,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.11846257001161575,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.11544708162546158,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.1051444336771965,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.0589425303041935,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.05600833147764206,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.06657960265874863,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.06164858862757683,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.059755317866802216,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.05331803858280182,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.050760675221681595,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.03372207656502724,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.029404275119304657,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02820514515042305,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.027920693159103394,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.01684894785284996,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.014409088529646397,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.014292186126112938,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.013249179348349571,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.01306531298905611,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.008753670379519463,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.00863049179315567,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.00835912860929966,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005466482602059841,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.46.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11148940026760101,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1048964112997055,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10214439779520035,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09303097426891327,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05219801887869835,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04952414706349373,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.059289757162332535,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05473075062036514,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05291486158967018,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04718557372689247,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04503446817398071,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.03008236363530159,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02613174170255661,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.024985890835523605,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.02471272647380829,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.01503722183406353,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.01278831996023655,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.012673788703978062,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.011767778545618057,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.011593765579164028,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007842238061130047,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007710036356002092,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007475849706679583,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004926753230392933,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.46.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.24622519314289093,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.2319110929965973,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.2272954285144806,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.20722970366477966,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.11558488756418228,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.11042217165231705,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12883272767066956,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11889217793941498,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.11712983250617981,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.10457257181406021,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.09964418411254883,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06539756804704666,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.056704048067331314,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05520814657211304,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.05486273765563965,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.032615888863801956,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.0280247014015913,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.027912328019738197,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.025726478546857834,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02550823614001274,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01674802228808403,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01634044572710991,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01618003658950329,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.010029052384197712,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.46.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.20305919647216797,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.16385610401630402,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.1468825340270996,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.1271439492702484,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.0925636813044548,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.07436173409223557,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11547111719846725,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10598662495613098,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.09703967720270157,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07007020711898804,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.06645320355892181,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.05935060232877731,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.050951212644577026,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04480414465069771,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04323972761631012,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.029713112860918045,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.023635856807231903,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.023052170872688293,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.019314978271722794,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.018226590007543564,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01587264984846115,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01596244052052498,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01386143546551466,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.011083945631980896,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.46.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.20392175018787384,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.19222474098205566,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.1886773258447647,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.17214716970920563,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.09593358635902405,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.09175534546375275,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.10644403845071793,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.0983729213476181,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.0971892774105072,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.08689358830451965,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.08281174302101135,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.05421340838074684,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.047093119472265244,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.045979615300893784,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04571999981999397,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.02713938057422638,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.023712094873189926,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.023639487102627754,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.0218817126005888,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.021717695519328117,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.014387249946594238,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014385884627699852,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.014027423225343227,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009670917876064777,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.46.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.24325484037399292,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.22932741045951843,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.2250709980726242,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.20536543428897858,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.11428192257881165,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.1093556135892868,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.1265660971403122,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.11716409772634506,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.11580131947994232,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.10356087982654572,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09858258068561554,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06434333324432373,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.055935680866241455,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.05463554710149765,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05432236194610596,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.03210468217730522,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.027808494865894318,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.0277237668633461,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.025584831833839417,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.025389809161424637,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.016644688323140144,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.016306141391396523,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.01621185429394245,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.010208054445683956,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.46.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.24948760867118835,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.2255554497241974,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.21579597890377045,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.1926124095916748,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.11453928053379059,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.10511095821857452,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.13507188856601715,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.12385376542806625,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.11803203821182251,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.10008516907691956,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.09428223967552185,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06829146295785904,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05927059426903725,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.05506230890750885,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.054033197462558746,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03433960676193237,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.028646891936659813,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.0283869206905365,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.02577841281890869,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.025133445858955383,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.01836097612977028,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.018220417201519012,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.016984228044748306,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.012377429753541946,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.47.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.1282060742378235,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.12048935890197754,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.11755339801311493,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.10698698461055756,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.06003936007618904,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.05700519680976868,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.06781841814517975,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.06268341094255447,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.060882892459630966,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.05422864109277725,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.0517253652215004,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.034429360181093216,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.029937002807855606,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02874014712870121,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.02845984883606434,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.01720385067164898,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.014718310907483101,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.014602822251617908,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.01352956797927618,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.013348262757062912,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.008975852280855179,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008865988813340664,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.008587177842855453,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005700818262994289,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.47.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11102709174156189,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.10435434430837631,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10156805068254471,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09248175472021103,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05198671296238899,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.049271807074546814,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.059166643768548965,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05461105331778526,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.052707575261592865,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.0469849556684494,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04487232491374016,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.030038421973586082,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.026074323803186417,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.024874253198504448,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.024592308327555656,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.0150113170966506,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.012717618606984615,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.01259528286755085,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.011693699285387993,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.01151324063539505,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007800381630659103,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007658487185835838,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007416907697916031,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004851412493735552,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.47.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.24666064977645874,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.23216667771339417,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.22751881182193756,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.20740404725074768,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.1158798560500145,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.11061563342809677,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12914277613162994,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11919619143009186,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.11741813272237778,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.10473274439573288,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.09972728788852692,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06559010595083237,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05688052624464035,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05535002425312996,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.054989270865917206,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.0326998196542263,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.028098221868276596,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.027979901060461998,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.025784729048609734,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02555590681731701,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01679357886314392,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.016403164714574814,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.0162142775952816,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.010067309252917767,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.47.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.18841895461082458,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.16760723292827606,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.16118073463439941,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.13767892122268677,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.0875958651304245,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.07994592189788818,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.10009792447090149,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.09198566526174545,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.09041456878185272,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07283395528793335,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.06648639589548111,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.05168726295232773,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.04441075772047043,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04237111657857895,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.0418759249150753,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.025990689173340797,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02236280031502247,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02226739004254341,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.019495483487844467,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.01916898600757122,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01425732858479023,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01448698341846466,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.013617880642414093,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.010493851266801357,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.47.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.19710472226142883,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.18581907451152802,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.18226689100265503,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.16621039807796478,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.09281256049871445,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.08874718099832535,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.10296301543712616,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.09517893195152283,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.09404164552688599,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.08402004837989807,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.08009465783834457,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.05247803032398224,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.04555363208055496,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.04448046535253525,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04422333464026451,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.02623436413705349,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02295745722949505,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.022884666919708252,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.021178200840950012,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02102251909673214,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.013874451629817486,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.013963505625724792,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.013527251780033112,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.00941451359540224,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.47.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.24128378927707672,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.22749392688274384,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.22320051491260529,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.20367518067359924,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.11348406970500946,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10856813937425613,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.12587963044643402,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.11632366478443146,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.1149619072675705,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.10275176167488098,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09796127676963806,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06405984610319138,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05558896064758301,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.054287005215883255,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05398058518767357,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.032000426203012466,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.027711624279618263,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.027628622949123383,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.025501767173409462,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.0253092460334301,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.016782166436314583,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.01638810522854328,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.016364658251404762,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.010445683263242245,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.47.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.23204126954078674,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.20781806111335754,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.19750402867794037,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.1766115427017212,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.1059085950255394,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.09621082991361618,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.1259433776140213,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11596410721540451,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.109588123857975,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09206889569759369,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08714593946933746,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06387468427419662,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.055307019501924515,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.05080248787999153,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.049688052386045456,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03193093091249466,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.02625383995473385,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.02595171332359314,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.023480895906686783,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.02276051603257656,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.016874903813004494,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.016587665304541588,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.015410490334033966,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.010857665911316872,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.48.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.1222965344786644,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.11496210843324661,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.11206474155187607,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.10196353495121002,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.0572790801525116,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.05437199026346207,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.06497685611248016,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.0598599910736084,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05808709189295769,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.05175301060080528,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.049409717321395874,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.032998502254486084,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.028591634705662727,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02741970494389534,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.027138754725456238,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.016488252207636833,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.01402976457029581,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.013918593525886536,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.012898550368845463,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.012724850326776505,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.008584316819906235,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008444493636488914,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.008204118348658085,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005405917763710022,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.48.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.10798198729753494,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.10149071365594864,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.09885471314191818,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.08997087180614471,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05050085112452507,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.0478835292160511,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05775903910398483,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05304436385631561,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.051213670521974564,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.045636136084795,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04373428598046303,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.02929675579071045,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.025340944528579712,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.024192893877625465,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.023920001462101936,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.014631032012403011,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.012381244450807571,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.012267395853996277,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.011387369595468044,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.011212946847081184,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007639700546860695,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007466987241059542,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007269874215126038,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.00475305737927556,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.48.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.24617359042167664,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.23165832459926605,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.22702008485794067,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.20692618191242218,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.11572113633155823,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.110430508852005,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12906351685523987,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11906265467405319,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.11730655282735825,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.10455350577831268,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.09959465265274048,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06561530381441116,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.056828923523426056,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05530313774943352,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.05495020002126694,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03273193538188934,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.028181517496705055,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02806522138416767,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.025875138118863106,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.025640001520514488,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01691165566444397,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.016602616757154465,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01633385941386223,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.010428712703287601,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.48.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.20721779763698578,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.17446687817573547,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.1638273298740387,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.13639037311077118,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.0951821431517601,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.08279623836278915,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11220042407512665,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10248531401157379,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.09907137602567673,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07371053099632263,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.06637430936098099,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.057746145874261856,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.04992020130157471,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04662798345088959,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04583252593874931,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.029185529798269272,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.025569450110197067,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.025356339290738106,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.02163882926106453,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02112705446779728,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.016460144892334938,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.017830820754170418,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.015475229360163212,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.013973583467304707,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.48.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.19882100820541382,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.187422975897789,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.18387164175510406,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.16775469481945038,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.09363744407892227,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.08956228196620941,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.10388335585594177,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.09605275839567184,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.09487856179475784,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.08476544916629791,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.0807717964053154,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.052959077060222626,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.04597114771604538,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.044876303523778915,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.044626615941524506,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.026440460234880447,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.023148374632000923,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.023075290024280548,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.021351872012019157,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02119026705622673,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.01396135613322258,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014048359356820583,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.013610144145786762,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009429828263819218,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.48.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.24184931814670563,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.2279701977968216,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.22373810410499573,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.20416657626628876,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.1137424185872078,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10882262885570526,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.126039519906044,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.11660974472761154,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.11522344499826431,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.1029830127954483,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.0981249213218689,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06410684436559677,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05568067356944084,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.054365698248147964,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.0540638193488121,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.0319950245320797,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.027704939246177673,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.027617253363132477,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.0254837553948164,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02528339996933937,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.016669396311044693,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.016293536871671677,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.01624368131160736,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.01026490144431591,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.48.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.23325690627098083,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.2099364697933197,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.20001806318759918,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.17887070775032043,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.10686325281858444,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.09741364419460297,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.1265089511871338,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11663362383842468,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.11037801951169968,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09325916320085526,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08814069628715515,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06436977535486221,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05591409653425217,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.051483795046806335,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.05038844794034958,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.0324123241007328,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.026990486308932304,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.026694118976593018,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.024321895092725754,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.023630758747458458,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.01737036369740963,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.017523810267448425,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.015916956588625908,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.012163910083472729,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.49.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.13100798428058624,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.12315753847360611,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.12020596861839294,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.10945754498243332,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.06139684468507767,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.058354564011096954,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.06908517330884933,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.0639142245054245,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.06225569546222687,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.055430151522159576,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.05277020111680031,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.035035114735364914,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.03049653023481369,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.029349934309720993,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.029090479016304016,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.017488082870841026,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.014982763677835464,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.014875349588692188,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.013754152692854404,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.013578956015408039,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.009061329998075962,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.00892838928848505,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.008684715256094933,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005622896831482649,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.49.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11454226821660995,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.10768011212348938,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10499550402164459,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09562650322914124,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05371363088488579,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.05100085586309433,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.060868024826049805,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05609873682260513,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05444248765707016,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04850858822464943,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.046327315270900726,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.0308938417583704,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.026783572509884834,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.0257040336728096,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.0254463329911232,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.015443005599081516,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.013128908351063728,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.013024288229644299,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.012065709568560123,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.011899491772055626,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.008022396825253963,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007860245183110237,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007661015726625919,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004980741534382105,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.49.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.2489214688539505,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.23437857627868652,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.22964416444301605,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.20931629836559296,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.11700694262981415,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.11172635108232498,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.13030299544334412,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.12030256539583206,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.11855723708868027,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.10573772341012955,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.10067135840654373,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06616489589214325,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05739184841513634,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.055902015417814255,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.05553877353668213,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03299683332443237,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.028400994837284088,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.028292398899793625,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.02606329321861267,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02583368308842182,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.016999799758195877,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01660754717886448,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.016441883519291878,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.010265296325087547,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.49.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.2060605138540268,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.18305039405822754,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.17491954565048218,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.1461000293493271,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.09561418741941452,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.08669254183769226,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11056427657604218,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10218051820993423,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.09837652742862701,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.0782865509390831,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.07104122638702393,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.056785862892866135,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.049058862030506134,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.046015117317438126,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.045276664197444916,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.028445670381188393,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.023926617577672005,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.023674746975302696,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.020467709749937057,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.019966108724474907,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.015173893421888351,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015132598578929901,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01418538112193346,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.010292691178619862,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.49.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.20236817002296448,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.19067583978176117,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.18708562850952148,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.17059603333473206,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.09531305730342865,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.0911610797047615,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.10569991916418076,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.09778617322444916,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.09658554196357727,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.08625742793083191,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.08217483758926392,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.053863584995269775,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.04680721461772919,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.04569358751177788,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04542986676096916,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.026912832632660866,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.0235360749065876,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.023461921140551567,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.0216955728828907,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.021531490609049797,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.014189798384904861,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014253377914428711,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.013825258240103722,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.00953284464776516,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.49.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.24375005066394806,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.22974222898483276,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.22539381682872772,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.2055778205394745,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.11462272703647614,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10964804887771606,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.12698405981063843,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.11755047738552094,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.11614751815795898,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.1037643551826477,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.098759725689888,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06457430124282837,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05611215904355049,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.05479959025979042,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.0544857457280159,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.03218717873096466,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02787167765200138,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.027785565704107285,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.025612687692046165,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.025418797507882118,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.0166849996894598,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.01632881909608841,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.016251595690846443,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.010187855921685696,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.49.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.24023906886577606,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.2166835069656372,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.2072056233882904,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.1850433051586151,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.11010558158159256,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.10084622353315353,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.12914466857910156,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11907544732093811,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.11354002356529236,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09601106494665146,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.09044719487428665,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.0655156597495079,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05680028721690178,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.05275043472647667,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.05175737291574478,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03276753053069115,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.027150653302669525,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.026890814304351807,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.024304939433932304,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.023664940148591995,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.01727396994829178,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.016881035640835762,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.01595205068588257,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.010913430713117123,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.50.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.12922215461730957,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.12136151641607285,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.11835483461618423,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.10760834813117981,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.06054958328604698,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.057475555688142776,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.0683986023068428,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.06322136521339417,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.06142021343111992,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.05463806912302971,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.05201445147395134,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.034711092710494995,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.030216217041015625,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.028987130150198936,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.028701744973659515,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.017355123534798622,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.014838282950222492,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.014719882979989052,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.013623335398733616,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.013435919769108295,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.009054077789187431,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.00892942026257515,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.008648766204714775,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.0057152993977069855,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.50.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.1127755343914032,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.10589205473661423,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10315301269292831,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09381041675806046,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.0528409443795681,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.050093065947294235,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.060054175555706024,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05538523569703102,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05359269306063652,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.047676458954811096,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.045471932739019394,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.030471447855234146,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02643951028585434,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02528488077223301,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.025005875155329704,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.015216129831969738,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.012926112860441208,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.012815533205866814,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.011867574416100979,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.011692960746586323,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007903669960796833,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007769932970404625,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007518287282437086,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004922306165099144,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.50.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.24918748438358307,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.23437833786010742,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.2296219766139984,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.2090667188167572,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.11706390231847763,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.1116776317358017,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.1305598020553589,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.1204548254609108,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.11863981932401657,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.10567603260278702,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.10056605935096741,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06627526879310608,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.057496000081300735,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05591503158211708,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.05554107204079628,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.033052168786525726,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.028357313945889473,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.028243178501725197,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.025997478514909744,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.025757893919944763,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.016967786476016045,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01653422974050045,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.016382558271288872,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.010108157992362976,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.50.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.21277645230293274,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.16276434063911438,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.14636166393756866,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.12319647520780563,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.0927778109908104,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.07495058327913284,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11824311316013336,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10579454898834229,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.09938506782054901,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07007468491792679,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.06543310731649399,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.05936321243643761,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.052797477692365646,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04699039086699486,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.045496560633182526,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.030658889561891556,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.027963608503341675,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.027584269642829895,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.024257926270365715,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.023426108062267303,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01806427724659443,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.02188718132674694,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.016330184414982796,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.018670734018087387,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.50.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.2070562243461609,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.1950092762708664,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.19128647446632385,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.17444676160812378,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.09760572016239166,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.09334076941013336,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.10851261019706726,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10014316439628601,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.09893067926168442,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.08830096572637558,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.08416147530078888,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.05536748468875885,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.04799548164010048,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.046851493418216705,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04658747836947441,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.027781788259744644,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.024261098355054855,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.02418651059269905,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02237497642636299,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02220791205763817,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.014949996024370193,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.01485995389521122,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.01458842121064663,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.010155811905860901,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.50.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.24697545170783997,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.23274068534374237,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.22830161452293396,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.20816266536712646,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.11624214053153992,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.11115824431180954,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.1291523575782776,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.11917523294687271,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.11776983737945557,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.10515087842941284,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.10019756853580475,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06576888263225555,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.056978676468133926,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.05563352629542351,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05532296746969223,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.0329228974878788,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.028454208746552467,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.028360944241285324,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.026169024407863617,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.025969501584768295,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.01743134669959545,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.016894297674298286,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.01700056716799736,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.010866777040064335,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.50.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.2396080493927002,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.21672706305980682,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.20713146030902863,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.18510600924491882,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.11004476249217987,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.10088963806629181,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.1291530430316925,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.11933200806379318,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.11341127753257751,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09629005938768387,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.09082356095314026,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06562166661024094,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.057144682854413986,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.05290871858596802,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.05185827985405922,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03284958004951477,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.02752642147243023,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.027240049093961716,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.02479691617190838,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.024137010797858238,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.01730320230126381,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.017552362754940987,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.015909411013126373,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.011879532597959042,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.51.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.13383163511753082,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1257837563753128,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.12285846471786499,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.11172153800725937,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.06279434263706207,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.059674132615327835,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.07060101628303528,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.06527146697044373,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.06369087845087051,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.056661464273929596,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.05389626696705818,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.03576973080635071,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.031169559806585312,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.030026381835341454,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.02975400909781456,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.017863180488348007,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.015299978666007519,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.015200559981167316,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.014031435362994671,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.013861047104001045,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.00923709012567997,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.00907363835722208,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.00885926466435194,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.00567882414907217,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.51.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11834593117237091,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.11125548928976059,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10848993808031082,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09870174527168274,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05550392344594002,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.05273010954260826,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.06273004412651062,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05789755657315254,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05629347637295723,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.050121862441301346,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.047805678099393845,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.03185281902551651,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.027635255828499794,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.026561567559838295,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.026306811720132828,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.01591007225215435,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.013556813821196556,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.013454907573759556,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.012441864237189293,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.012279982678592205,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.008243050426244736,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008082438260316849,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007887545973062515,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005087659228593111,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.51.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.25406330823898315,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.23902039229869843,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.2342582643032074,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.21330679953098297,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.11946994811296463,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.11402194201946259,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.1329881101846695,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.1228116899728775,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.12109420448541641,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.10785941034555435,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.10263881087303162,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06755941361188889,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.0586211122572422,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05706903710961342,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.056707002222537994,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03367673233151436,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.028952166438102722,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02883644588291645,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.02654295787215233,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02630803734064102,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.017305031418800354,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.016860580071806908,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.016737014055252075,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.010303480550646782,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.51.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.22283092141151428,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.18227024376392365,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.16865451633930206,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.14652840793132782,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10193087160587311,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.0857929065823555,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11978363990783691,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11029458791017532,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10658850520849228,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07760129868984222,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.07326457649469376,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.061382658779621124,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.053004201501607895,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04918074607849121,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.048254575580358505,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.030799800530076027,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.025704506784677505,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.025468895211815834,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.020911991596221924,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02025873027741909,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01659291237592697,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.016535023227334023,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.015389811247587204,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.01141280122101307,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.51.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.21164503693580627,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.1993085741996765,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.19547319412231445,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.17816443741321564,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.09977023303508759,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.09536310285329819,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11079084128141403,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10234776884317398,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.10112327337265015,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09019321203231812,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.08602750301361084,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.05647950619459152,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.049007728695869446,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.04783177375793457,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04755586385726929,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.028247883543372154,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.024704143404960632,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.02462472766637802,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.022759417071938515,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02258855476975441,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.014965962618589401,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.01503832172602415,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.014587825164198875,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.010154826566576958,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.51.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.24982154369354248,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.23523136973381042,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.23075911402702332,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.2102661281824112,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.11756569892168045,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.11237125098705292,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.13025131821632385,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.12056316435337067,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.11912145465612411,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.10625303536653519,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.10112529993057251,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06627009063959122,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05759049952030182,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.05620596557855606,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05589069053530693,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.03305593878030777,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02863556146621704,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.028545072302222252,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.026304425671696663,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.026094486936926842,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.017181694507598877,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.01684919185936451,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.016732603311538696,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.010595858097076416,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.51.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.24621886014938354,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.22284193336963654,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.21330814063549042,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.19040749967098236,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.11313053220510483,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.10392177104949951,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.13277997076511383,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.12215444445610046,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.11650333553552628,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09889425337314606,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.09322387725114822,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.0674891397356987,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05832798779010773,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.0542435422539711,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.05325036123394966,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03394698724150658,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.027988310903310776,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.027725012972950935,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.025128517299890518,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.02448412775993347,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.018207216635346413,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.017486967146396637,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.016869544982910156,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.011443942785263062,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.52.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.13923464715480804,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.13077247142791748,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.12771032750606537,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.11614644527435303,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.06535597145557404,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.062111664563417435,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.07362028956413269,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.06796237081289291,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.0662611573934555,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.05892946571111679,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.05608290433883667,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.037372298538684845,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.032441407442092896,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.03125820681452751,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.03098195604979992,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.018667323514819145,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.01596059650182724,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.015852659940719604,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.014636932872235775,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.01445647981017828,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.009695724584162235,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.009507868438959122,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.00928846001625061,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.006003342568874359,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.52.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.12243567407131195,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.11508385092020035,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.11223588138818741,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.10209088027477264,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05749812722206116,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.05459010228514671,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.06508512049913406,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05999766290187836,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05831748992204666,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.051880184561014175,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04947735369205475,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.033073775470256805,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.028658874332904816,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.027526484802365303,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.027260195463895798,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.016515536233782768,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.014061548747122288,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.013955770060420036,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.012905755080282688,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.012738203629851341,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.008579837158322334,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008410154841840267,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.008204499259591103,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005325671285390854,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.52.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.2553623914718628,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.24014467000961304,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.23540954291820526,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.21418419480323792,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.12012030929327011,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.11460620164871216,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.13369469344615936,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.1234450414776802,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.12172766029834747,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.10837097465991974,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.10309498757123947,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06789334863424301,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05892014130949974,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05738789215683937,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.057014692574739456,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03386187553405762,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.029141297563910484,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.029035421088337898,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.02670709602534771,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.026469022035598755,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01743137650191784,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.017017416656017303,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.016864337027072906,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.010476615279912949,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.52.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.23329992592334747,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.2012554407119751,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.19083698093891144,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.15684977173805237,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.10700474679470062,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.09500865638256073,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12490017712116241,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11515724658966064,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.11178189516067505,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.08554565161466599,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.0767592340707779,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06407728791236877,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.055340725928545,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05163779854774475,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.05073919892311096,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03203834593296051,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02704612724483013,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.026832079514861107,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.022840868681669235,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.022220049053430557,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01713504083454609,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01743662729859352,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01594284363090992,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.012155342847108841,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.52.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.20917534828186035,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.19682663679122925,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.19305701553821564,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1758374124765396,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.09864186495542526,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.09423452615737915,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.10945052653551102,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10122311860322952,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.09996728599071503,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.08911482989788055,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.084851935505867,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.05583658814430237,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.04849085956811905,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.04731029272079468,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04703669622540474,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.0279035996645689,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.024437732994556427,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.02435530163347721,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02250373736023903,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.022331630811095238,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.014759027399122715,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014885969460010529,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.014377548359334469,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.010054298676550388,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.52.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.25128525495529175,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.2365458458662033,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.2319718599319458,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.21135887503623962,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.11832931637763977,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.11309418827295303,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.13138942420482635,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.12142187356948853,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.11995232850313187,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.10692975670099258,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.10178586840629578,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.0669729933142662,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.058068983256816864,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.056669387966394424,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05633736401796341,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.03343838080763817,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.029001066461205482,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.028909264132380486,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.026652351021766663,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02644795924425125,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.017566286027431488,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.017266878858208656,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.017105484381318092,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.011146110482513905,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.52.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.25137272477149963,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.22773605585098267,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.2181008756160736,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.19481882452964783,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.11567751318216324,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.10630790889263153,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.1351638287305832,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.12488637864589691,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.11901243776082993,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.10120517760515213,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.09519774466753006,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06858688592910767,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.059714604169130325,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.05557692050933838,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.05456581339240074,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03435397148132324,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.0288546122610569,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.028584348037838936,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.02597443386912346,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.02533251792192459,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.018193114548921585,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.018266800791025162,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.016872825101017952,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.012297812849283218,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.53.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.13633474707603455,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1279955506324768,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.12485016882419586,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.11346613615751266,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.06398167461156845,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.060738079249858856,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.0720835030078888,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.06668698042631149,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.06488248705863953,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.057623885571956635,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.05478871613740921,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.0365619882941246,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.03182859718799591,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.030617009848356247,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.030322764068841934,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.018260104581713676,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.01562768593430519,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.01551580149680376,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.014319201000034809,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.014134140685200691,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.009469714015722275,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.00932313222438097,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.009067261591553688,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005886521656066179,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.53.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.12051814794540405,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.11311786621809006,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.11024831980466843,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.10019414126873016,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05655158311128616,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.053663432598114014,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.06403527408838272,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.059080205857753754,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05736842378973961,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.05096719041466713,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04855011776089668,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.032512206584215164,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02822212502360344,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.027074022218585014,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.026803886517882347,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.016242187470197678,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.013844233006238937,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.013734035193920135,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.012696397490799427,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.0125228026881814,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.00843978300690651,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008305170573294163,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.008064961060881615,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005281426012516022,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.53.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.257175475358963,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.24173864722251892,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.23684574663639069,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.21543022990226746,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.12090309709310532,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.11535584181547165,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.13462448120117188,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.12439900636672974,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.12259955704212189,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.10900656878948212,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.10362651944160461,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.0683773010969162,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05934528261423111,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05777449905872345,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.05739646777510643,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03409247845411301,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.029313305392861366,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.029201341792941093,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.026841670274734497,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02659912221133709,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01752181351184845,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.017087293788790703,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01694532297551632,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.010467953979969025,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.53.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.21101199090480804,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.17294040322303772,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.15689580142498016,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.13563886284828186,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.09668519347906113,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.0789395347237587,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11908258497714996,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.1094532310962677,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10099733620882034,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07468590885400772,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.06945180147886276,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06114520877599716,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05262273550033569,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.046809859573841095,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.045346058905124664,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.030615154653787613,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.024741053581237793,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.024179788306355476,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.02049267664551735,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.019477801397442818,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.016446948051452637,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.016618456691503525,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.014582693576812744,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.011638445779681206,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.53.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.21014772355556488,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.19768549501895905,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.19381064176559448,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.17645739018917084,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.09908416867256165,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.09462842345237732,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.1100350096821785,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10173065215349197,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.10045424103736877,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.08943498134613037,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.08508411794900894,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.056153345853090286,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.04872190207242966,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.047529660165309906,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04723835736513138,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.02810570038855076,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.024533530697226524,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.024450700730085373,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.022579263895750046,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.022400308400392532,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.014929266646504402,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.01492883451282978,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.014541417360305786,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.010060506872832775,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.53.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.253106951713562,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.23808754980564117,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.23350635170936584,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.21251317858695984,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.11914245039224625,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.11380082368850708,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.13211487233638763,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.12227068841457367,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.12077057361602783,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.10752151161432266,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.10219976305961609,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06721893697977066,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05839421972632408,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.056977514177560806,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05663925036787987,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.0335259735584259,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02900831215083599,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.028918392956256866,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02660590410232544,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.026392055675387383,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.017410513013601303,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.017043953761458397,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.01694166287779808,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.010680273175239563,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.53.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.25837695598602295,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.2344224601984024,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.22482722997665405,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.20061929523944855,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.11904137581586838,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.1096351221203804,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.1395452320575714,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.1280411332845688,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.12241166085004807,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.10417648404836655,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.09798134863376617,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.07056546211242676,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.06124579533934593,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.05718115344643593,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.056187309324741364,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.035430531948804855,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.029664792120456696,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.029408887028694153,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.02669603005051613,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.026065543293952942,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.018883265554904938,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.018709318712353706,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.017548810690641403,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.012581191025674343,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.54.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.1404666304588318,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.13179421424865723,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.12853695452213287,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.11670675873756409,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.06593768298625946,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.06256519258022308,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.07433930039405823,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.06874962151050568,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.0668763816356659,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.05932704731822014,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.05635648965835571,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.03772791475057602,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.032814618200063705,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.03156188130378723,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.03125801682472229,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.018833650276064873,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.01611713320016861,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.016000160947442055,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.014756354503333569,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.014567852020263672,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.009763031266629696,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.009634618647396564,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.009340325370430946,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.006100093945860863,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.54.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.12254678457975388,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.11497699469327927,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.11202351748943329,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.10177026689052582,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05754714459180832,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.054549690335989,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.06531533598899841,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.06021364405751228,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.058367032557725906,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.05181632190942764,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.049375154078006744,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.03317039832472801,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.028761887922883034,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02755047008395195,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.02726900391280651,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.016568969935178757,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.014083024114370346,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.013963905163109303,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.012910065241158009,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.012723972089588642,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.00861161109060049,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008453351445496082,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.00821451935917139,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005362134892493486,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.54.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.2635132074356079,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.2475944310426712,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.24253469705581665,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.2203904539346695,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.1239439994096756,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.11822275072336197,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.13807033002376556,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.12752890586853027,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.1257062405347824,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.11166750639677048,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.10603296011686325,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.07019355893135071,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.06087638437747955,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05926814675331116,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.058865319937467575,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03498278558254242,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.030076589435338974,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.029958095401525497,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.02750731259584427,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.027262119576334953,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.017988432198762894,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.017540279775857925,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.017388110980391502,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.010759103111922741,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.54.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.22104203701019287,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.18494801223278046,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.17157495021820068,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.13377663493156433,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.09942226111888885,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.08557794988155365,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12093204259872437,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11121102422475815,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10569699853658676,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07566487044095993,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.06791801750659943,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06183511018753052,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.0533229261636734,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04791663587093353,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04656735435128212,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.030896436423063278,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.025003677234053612,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02463160827755928,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.0203190129250288,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.019368600100278854,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.016340279951691628,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01630108244717121,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.014591291546821594,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.010973130352795124,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.54.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.2091469168663025,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.19662430882453918,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.19279101490974426,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1754305511713028,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.09865792840719223,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.09417346119880676,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.10950945317745209,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.1012997105717659,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.10003627836704254,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.08896463364362717,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.08456485718488693,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.05585242807865143,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.04853137955069542,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.047332197427749634,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.047045838087797165,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.027926167473196983,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02442779205739498,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.02434779517352581,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.022464413195848465,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02228657901287079,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.014743326231837273,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014860240742564201,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.01435129251331091,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.01000587921589613,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.54.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.25243207812309265,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.23739564418792725,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.23279091715812683,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.2118607461452484,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.1189783439040184,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.11358802020549774,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.1319030374288559,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.12208565324544907,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.12058697640895844,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.1073000580072403,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.10196012258529663,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06714148074388504,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05834529548883438,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.056916650384664536,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05657358467578888,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.03350875526666641,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.029019925743341446,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.028927963227033615,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02661391720175743,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02639576978981495,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.017413778230547905,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.01711425371468067,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.01694299653172493,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.010823594406247139,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.54.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.25489091873168945,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.2306220680475235,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.2208702117204666,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.1970241218805313,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.11722797155380249,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.10773295909166336,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.1375337541103363,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.12656083703041077,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.12073419988155365,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.10248108208179474,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.09649255871772766,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06998321413993835,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.060481954365968704,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.05623840168118477,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.05520845949649811,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.035232290625572205,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.029049420729279518,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.028784150257706642,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.026094600558280945,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.02542693354189396,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.01898834854364395,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.01819406822323799,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.01762227714061737,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.011974208056926727,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.55.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.13802015781402588,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.12948012351989746,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.1261601448059082,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.11453711241483688,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.06481829285621643,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.06145191937685013,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.07326273620128632,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.06770417094230652,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.06574388593435287,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.058324653655290604,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.05543533340096474,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.03717482462525368,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.032345958054065704,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.03102801740169525,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.03071490488946438,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.018576564267277718,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.015858875587582588,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.015734849497675896,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.014519869349896908,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.01431929413229227,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.009645931422710419,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.00949960108846426,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.009204389527440071,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.006022167857736349,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.55.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.12161269038915634,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.11403750628232956,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.11106015741825104,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.10084368288516998,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.057083290070295334,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.054100193083286285,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.0646718367934227,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.059724047780036926,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05789447948336601,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.05136289447546005,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04885832220315933,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.0328424870967865,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.028531676158308983,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02732272259891033,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.027035273611545563,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.016411662101745605,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.013966546393930912,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.013852396048605442,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.012793663889169693,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.01261055562645197,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.00852205976843834,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008382480591535568,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.008128290995955467,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005321484059095383,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.55.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.25900277495384216,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.2432720959186554,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.2382626235485077,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.21646584570407867,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.12185432761907578,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.11613592505455017,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.13569512963294983,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.12544704973697662,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.12357208132743835,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.10974257439374924,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.10420309752225876,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06892059743404388,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05986576899886131,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05821720138192177,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.057835765182971954,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03436821699142456,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02953645959496498,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02941872552037239,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.027011046186089516,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02675679884850979,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01763911359012127,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01720644161105156,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01704469695687294,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.010501768440008163,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.55.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.21658052504062653,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.18501049280166626,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.17354807257652283,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.15152058005332947,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.09935150295495987,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.08656950294971466,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11836601048707962,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10786698013544083,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.10319837182760239,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07973941415548325,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.0745847076177597,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06048084795475006,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.052129101008176804,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.048283446580171585,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04735706001520157,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.030522678047418594,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.025905385613441467,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02563592791557312,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.02218456007540226,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.021571582183241844,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01703154295682907,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.017520645633339882,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.015874110162258148,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.013083093799650669,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.55.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.207818403840065,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.19541442394256592,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.19153280556201935,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1742311418056488,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.09804828464984894,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.09357400983572006,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.10911794006824493,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10068821161985397,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.09938745200634003,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.08842139691114426,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.08408676832914352,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.05566570535302162,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.04828329756855965,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.04707185551524162,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04678577184677124,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.027829192578792572,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02436070889234543,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.024280713871121407,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.022418536245822906,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.022241555154323578,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.014787116087973118,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014926978386938572,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.014398635365068913,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.010177118703722954,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.55.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.25120478868484497,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.23617136478424072,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.23150666058063507,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.2107052505016327,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.11838896572589874,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.11299710720777512,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.13167227804660797,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.12155087292194366,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.12005504965782166,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.10673893988132477,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.10147783905267715,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06710602343082428,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05814150720834732,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.05671704560518265,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05637283995747566,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.0335315465927124,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.029036205261945724,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.028940968215465546,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.026642829179763794,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.026427995413541794,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.01770343817770481,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.017310546711087227,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.01723908819258213,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.011206712573766708,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.55.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.2559488117694855,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.23141083121299744,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.22115464508533478,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.19714978337287903,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.11773351579904556,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.10793011635541916,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.13887032866477966,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.12775270640850067,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.12138738483190536,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.1028430312871933,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.09676949679851532,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.07051075249910355,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.06111934781074524,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.0565565824508667,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.055437467992305756,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03554585576057434,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.029329296201467514,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.029021447524428368,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.026354240253567696,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.025632990524172783,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.019171763211488724,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.018569570034742355,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.017723003402352333,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.012386785820126534,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.56.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.1396833211183548,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.13093706965446472,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.12754304707050323,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.11565160006284714,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.06556278467178345,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.0621153861284256,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.07413060963153839,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.06858336925506592,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.06655167043209076,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.058933816850185394,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.055959925055503845,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.037584200501441956,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.03274361044168472,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.03139578551054001,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.03106991946697235,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.01877286098897457,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.0160568468272686,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.015930235385894775,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.014696585945785046,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.014490881934762001,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.009758704341948032,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.009647374041378498,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.009316574782133102,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.006138452794402838,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.56.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.12070034444332123,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.11310970038175583,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.11005708575248718,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.0998196080327034,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.056657079607248306,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.053573694080114365,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.06452417373657227,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05951876565814018,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05748631805181503,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.05093855410814285,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04848309978842735,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.032767120748758316,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.028449995443224907,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02714511938393116,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.02683403342962265,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.016383372247219086,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.013915757648646832,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.013785827904939651,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.012748335488140583,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.012550598941743374,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.00853890459984541,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008426888845860958,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.008113845251500607,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005411481950432062,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.56.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.26218435168266296,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.24611610174179077,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.24101440608501434,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.2189488708972931,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.12339361757040024,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.11756811290979385,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.13755621016025543,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.1270371377468109,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.12514744699001312,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.11103315651416779,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.10541055351495743,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06986287236213684,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.060634780675172806,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.058971717953681946,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.0585721880197525,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.034842900931835175,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02993643283843994,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02981739304959774,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.027361836284399033,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.027106650173664093,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01789395697414875,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.017470907419919968,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.017280612140893936,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.010713709518313408,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.56.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.1983741968870163,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.16625328361988068,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.15351685881614685,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.128526508808136,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.0892479196190834,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.07629214972257614,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.10968287289142609,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10113052278757095,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.0946735218167305,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07077337056398392,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.06515496969223022,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.05605036020278931,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.048411834985017776,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04302060976624489,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04165136441588402,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.028021955862641335,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.022446146234869957,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.022013811394572258,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.018806947395205498,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.017866283655166626,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01481948047876358,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.014747122302651405,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01306694746017456,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.009855018928647041,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.56.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.20793753862380981,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.19551564753055573,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.1916150450706482,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.17427748441696167,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.09815634042024612,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.09364993125200272,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.10902601480484009,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10080289840698242,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.09949082136154175,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.08844810724258423,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.08406376838684082,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.0556313619017601,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.0483001209795475,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.04709051176905632,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04681381955742836,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.02780875377357006,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.024342846125364304,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.02426118776202202,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02238185703754425,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.022198941558599472,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.014729016460478306,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014860962517559528,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.014337231405079365,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.01007312536239624,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.56.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.24440252780914307,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.22978606820106506,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.2251848429441452,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.20487694442272186,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.11519357562065125,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10993735492229462,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.12800320982933044,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.11825428903102875,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.11677820980548859,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.1038242056965828,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09867808222770691,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06524071097373962,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05655644088983536,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.055158231407403946,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.05483248829841614,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.03259248659014702,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.028191519901156425,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.028097622096538544,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.025853067636489868,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.025641564279794693,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.017116256058216095,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.016738703474402428,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.01665603742003441,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.010722516104578972,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.56.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.2551003694534302,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.23004910349845886,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.2197527289390564,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.1958395391702652,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.11712294071912766,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.10728630423545837,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.13780823349952698,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.12719084322452545,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.12093570083379745,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.10219546407461166,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.09609758108854294,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.07005016505718231,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.06076710671186447,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.0562555305659771,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.05515223741531372,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.035048261284828186,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.029136359691619873,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.02885175123810768,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.026152057573199272,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.02544371597468853,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.0185228418558836,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.01840486191213131,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.017051301896572113,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.012220741249620914,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.57.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.13760870695114136,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.12898796796798706,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.12565703690052032,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.11394160985946655,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.06459121406078339,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.06114521250128746,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.07300186902284622,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.06759192794561386,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.0655335783958435,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.058042485266923904,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.05506633594632149,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.0370626337826252,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.03228599205613136,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.030923526734113693,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.030597595497965813,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.018499331548810005,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.01582321897149086,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.015688832849264145,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.014473466202616692,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.014266316778957844,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.009616885334253311,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.00951464381068945,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.009171999990940094,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.006061370950192213,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.57.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11822392046451569,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.11083517223596573,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10779064893722534,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09780925512313843,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05547698959708214,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.05247838422656059,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.06310466676950455,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05828550085425377,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.05629827827215195,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.049878817051649094,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04744700714945793,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.03204570710659027,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02783602848649025,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.026583081111311913,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.026282351464033127,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.016017433255910873,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.013626509346067905,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.01350054144859314,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.012485321611166,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.012293435633182526,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.008353067561984062,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008251768536865711,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007944895885884762,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005306532606482506,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.57.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.2615777552127838,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.2455929070711136,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.24049146473407745,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.2183839976787567,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.12314502894878387,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.11732614040374756,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.1372540444135666,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.12676115334033966,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.1248655766248703,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.11081050336360931,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.10516475141048431,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06969774514436722,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.06051904708147049,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.0588625967502594,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.05846862867474556,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.034783490002155304,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.029872987419366837,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.029757214710116386,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.02730780653655529,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02705605886876583,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.017875058576464653,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.017445361241698265,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.017264889553189278,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.010701396502554417,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.57.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.2013290375471115,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.17180712521076202,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.1625758409500122,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.13221758604049683,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.09290111809968948,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.08185536414384842,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.10717455297708511,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.09854812175035477,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.09621606767177582,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07166456431150436,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.06527481973171234,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.05492609739303589,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.04729815945029259,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.044717274606227875,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.04409165307879448,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.02747713401913643,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02329767681658268,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.023151107132434845,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.01911945268511772,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.01867624931037426,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.014727992936968803,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.014736076816916466,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.013894310221076012,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.010146019980311394,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.57.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.20716391503810883,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.19470040500164032,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.19081982970237732,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.17352747917175293,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.09775732457637787,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.09324390441179276,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.10855259746313095,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10040947794914246,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.0991140827536583,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.08804653584957123,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.08366521447896957,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.055318113416433334,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.04810648411512375,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.04688424989581108,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04659980535507202,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.027645573019981384,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02420886792242527,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.024124955758452415,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.022244315594434738,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.0220639668405056,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.014561320655047894,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014737214893102646,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.014165994711220264,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009938138537108898,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.57.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.2384471446275711,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.22405943274497986,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.21963337063789368,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.19974873960018158,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.11239089071750641,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.1072247251868248,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.12471281737089157,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.1154068261384964,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.11394145339727402,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.10125455260276794,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09617043286561966,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.06353946030139923,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.0551581047475338,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.05378485843539238,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.053453218191862106,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.03169211372733116,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02745114266872406,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.027359813451766968,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.025148021057248116,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.024942340329289436,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.016497883945703506,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.016230188310146332,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.01604137010872364,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.01030061300843954,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.57.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.25580233335494995,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.2296779602766037,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.2191455066204071,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.19516533613204956,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.11721992492675781,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.10700424760580063,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.13814660906791687,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.12743058800697327,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.1213253065943718,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.10196980834007263,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.0958310067653656,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.07019820809364319,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.06086176633834839,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.056256335228681564,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.05513418838381767,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03510350361466408,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.02906552329659462,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.028788557276129723,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.02600017748773098,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.025272220373153687,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.018501367419958115,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.01827561855316162,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.01699494756758213,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.011988845653831959,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.58.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.13371506333351135,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.12511445581912994,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.12159431725740433,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.11012684553861618,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.06265834718942642,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.05915995314717293,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.07123115658760071,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.065957210958004,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.06360410898923874,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.056256603449583054,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.05337687209248543,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.036153391003608704,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.031501904129981995,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.030018851161003113,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.029663344845175743,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.018079888075590134,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.015370387583971024,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.015223338268697262,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.014055510051548481,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.013829641975462437,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.009423515759408474,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.00930611789226532,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.008948871865868568,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005952071398496628,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.58.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11196256428956985,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.10475500673055649,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10162298381328583,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09206097573041916,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05243081599473953,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04943302273750305,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.060024090111255646,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05545235797762871,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.053209200501441956,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04708048701286316,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04472072049975395,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.030449053272604942,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02647881954908371,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.025117022916674614,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.02479046955704689,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.015227232128381729,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.012888583354651928,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.012747073546051979,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.011795291677117348,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.011584998108446598,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.00796184316277504,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007851249538362026,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.0075270733796060085,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005048719700425863,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.58.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.25814956426620483,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.24222280085086823,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.23715326189994812,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.21517442166805267,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.12150026857852936,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.11568702012300491,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.1357380449771881,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.12521758675575256,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.1232777088880539,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.10926096141338348,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.10369884967803955,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06899866461753845,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05981261655688286,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05809979513287544,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.057701475918293,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03442323952913284,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.029546765610575676,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02941901609301567,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.027007512748241425,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.02674749121069908,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.017715753987431526,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.017331091687083244,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01708577200770378,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.010737203992903233,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.58.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.20473766326904297,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.17142711579799652,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.16001565754413605,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.13258090615272522,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.0930807888507843,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.08093730360269547,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.11101261526346207,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.10194799304008484,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.09815748780965805,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.07273884117603302,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.06588919460773468,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.05693262070417404,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.04905159771442413,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.04507763311266899,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.044110603630542755,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.02863885462284088,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.023919185623526573,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.023684898391366005,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.0200161412358284,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.019352547824382782,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.015549763105809689,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.015926863998174667,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01428825594484806,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.011484690941870213,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.58.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.1997317671775818,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.1876634806394577,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.18384391069412231,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.16717635095119476,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.09426884353160858,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.08989039063453674,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.1048494502902031,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.09688140451908112,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.09559090435504913,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.08487547188997269,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.08060051500797272,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.05348554253578186,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.04643716290593147,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.04524531960487366,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.0449603796005249,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.026756800711154938,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02339175157248974,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.023308254778385162,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02149568684399128,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.02131827548146248,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.01418050192296505,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014293478801846504,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.013781108893454075,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009684762917459011,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.58.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.2260991781949997,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.21249158680438995,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.20821037888526917,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.18934640288352966,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.10665436089038849,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.10171812772750854,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.11852508038282394,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10954771190881729,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.1081179529428482,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.09602902084589005,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.09125121682882309,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.060378558933734894,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.05237344279885292,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.0510527566075325,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.050740357488393784,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.030133455991744995,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.026099929586052895,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.026002991944551468,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.023915497586131096,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.023715922608971596,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.015752827748656273,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.015496745705604553,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.0153127396479249,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009936205111443996,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.58.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.24800513684749603,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.22228337824344635,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.2121063470840454,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.18858768045902252,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.11382608860731125,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.10385838150978088,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.1346004605293274,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.12347548454999924,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.11783626675605774,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09872020781040192,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.09275533258914948,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.06846059113740921,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.059104178100824356,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.05475786700844765,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.05369957536458969,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.03431924805045128,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.028544215485453606,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.028292696923017502,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.025549964979290962,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.024868372827768326,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.01835629902780056,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.018256057053804398,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.016923414543271065,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.01243807002902031,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.59.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.12683595716953278,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.11867375671863556,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.11520137637853622,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.1042388379573822,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05944383144378662,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.056052155792713165,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.0679570883512497,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.0627702996134758,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.06033705174922943,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.05330368131399155,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.050649795681238174,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.034534864127635956,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.030012916773557663,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.028520409017801285,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.028160104528069496,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.01728089340031147,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.014692348428070545,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.014540163800120354,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.013446132652461529,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.013220326974987984,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.00908690970391035,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.009034695103764534,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.008619504980742931,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005951095372438431,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.59.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.10909169912338257,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.10206600278615952,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.09886831790208817,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.08946261554956436,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.0511210635304451,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04809724539518356,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.058828871697187424,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05434468016028404,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.051888711750507355,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04584025964140892,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04353680461645126,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.02987595833837986,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.0259799025952816,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02451913058757782,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.024169335141777992,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.014954831451177597,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.012631543911993504,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.012479530647397041,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.011559720151126385,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.011336265131831169,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.007857079617679119,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007788810413330793,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007393315900117159,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005100890062749386,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.59.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.24757598340511322,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.23210954666137695,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.22695466876029968,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.20566943287849426,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.11652549356222153,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.110809825360775,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.13039645552635193,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.12030531466007233,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.11822281777858734,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.10452403873205185,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.09919438511133194,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06629973649978638,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05747160688042641,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.055726557970047,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.0553198903799057,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03307463973760605,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.028360703960061073,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.028227534145116806,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.02588595077395439,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.025617940351366997,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.01707156002521515,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01670495979487896,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.016445789486169815,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.01040662731975317,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.59.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.14084656536579132,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1207309439778328,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.11435516923666,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09500426799058914,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.06307902932167053,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.056466661393642426,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.07523169368505478,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.06851281225681305,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.06687915325164795,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.051373519003391266,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04690089076757431,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.03855625167489052,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.033726613968610764,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.031290724873542786,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.030703207477927208,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.019608784466981888,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.017780840396881104,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.01768329367041588,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.01575651206076145,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.015400678850710392,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.011331073008477688,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.013100001029670238,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.01063112448900938,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.010811705142259598,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.59.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.19008775055408478,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.17856848239898682,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.17493197321891785,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1589038223028183,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.08969585597515106,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.08551707118749619,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.09980031102895737,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.09222454577684402,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.09096559137105942,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.08077903836965561,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.07673553377389908,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.05097809061408043,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.04425281658768654,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.0430930070579052,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.04281488433480263,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.025510024279356003,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.022358406335115433,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.022275201976299286,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.02056174725294113,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.020392652601003647,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.013588597998023033,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.01377808302640915,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.013205530121922493,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009479038417339325,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.59.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.2094571441411972,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.1967363953590393,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.19281986355781555,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.17522363364696503,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.09876088052988052,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.09418071806430817,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.10990942269563675,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.10151374340057373,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.1001351922750473,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.08890851587057114,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.08443789184093475,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.056099314242601395,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.04858209192752838,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.0473332554101944,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.047037042677402496,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.028021641075611115,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.024303659796714783,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.024219105020165443,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.022291623055934906,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.022105732932686806,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.014804637059569359,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014597397297620773,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.014386932365596294,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009569978341460228,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.59.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.23901066184043884,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.2131224274635315,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.203121155500412,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.18044538795948029,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.10968628525733948,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.09968394041061401,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.12999500334262848,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.1189388632774353,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.11372558772563934,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.09461455047130585,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.08892744779586792,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.0663570687174797,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.05717092752456665,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.05298035219311714,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.051964156329631805,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.033452559262514114,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.027992108836770058,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.027777191251516342,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.025066528469324112,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.02441800944507122,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.018359296023845673,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.018380286172032356,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.01703735813498497,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.01312380563467741,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.60.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.12441965192556381,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.1164446473121643,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.11298506706953049,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.10232710838317871,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.058342739939689636,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.055045753717422485,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.06696633249521255,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.061682648956775665,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.059209395200014114,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.052368372678756714,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04988238960504532,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.034010328352451324,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.029499219730496407,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.028002696111798286,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.027643023058772087,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.017050547525286674,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.01442563347518444,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.01427065022289753,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.013212556019425392,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.012983565218746662,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.008990532718598843,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008870404213666916,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.0085253631696105,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.0058240038342773914,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.60.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.10500133037567139,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.09826155006885529,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.0950789824128151,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.08610444515943527,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.049170590937137604,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04624057188630104,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05673951283097267,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.05241630598902702,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.04990409314632416,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.0441368967294693,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.041955314576625824,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.02879646047949791,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02505793794989586,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.023594941943883896,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.023238366469740868,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.014425136148929596,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.012161191552877426,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.0120000708848238,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.011141425929963589,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.010917345061898232,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.0076036653481423855,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.007520588580518961,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.007143748924136162,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004930283408612013,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.60.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.25233253836631775,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.23655018210411072,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.23153646290302277,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.20981404185295105,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.1187586560845375,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.11300871521234512,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.1327388435602188,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.12245891243219376,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.12050113826990128,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.10662879794836044,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.10120423138141632,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06748664379119873,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05850161612033844,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.0568050891160965,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.05640991032123566,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03365727514028549,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.02891397848725319,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02878863550722599,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.026402171701192856,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.026141991838812828,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.017343949526548386,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.01702018827199936,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.016716215759515762,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.010621304623782635,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.60.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.15494443476200104,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.12827156484127045,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.12048026919364929,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09699704498052597,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.07077480852603912,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.06122836843132973,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.08262092620134354,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.07510612159967422,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.07412842661142349,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.05330894514918327,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04867244139313698,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.042668480426073074,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.036733031272888184,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.03484553471207619,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.03437698632478714,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.021881338208913803,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.019361909478902817,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.019294194877147675,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.016271134838461876,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.01596873253583908,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.012801586650311947,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.013719666749238968,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.012254344299435616,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.011064018122851849,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.60.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.15685784816741943,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.1471838504076004,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.14421717822551727,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.13097739219665527,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.07427838444709778,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07084470242261887,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.08262362331151962,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.07638292014598846,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.07536165416240692,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.06697233021259308,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.06369749456644058,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.0426393486559391,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.03736906498670578,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.0364302396774292,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.03621252626180649,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.021541880443692207,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.020122196525335312,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.020057599991559982,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.018780289217829704,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.01865271106362343,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.012259487062692642,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.013989712111651897,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.011979123577475548,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.011320012621581554,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.60.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.1671953946352005,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.15706215798854828,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.15384899079799652,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1399828940629959,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.07932525128126144,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.07565085589885712,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.08849894255399704,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.08151955902576447,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.08042658865451813,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.07148043811321259,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.06809869408607483,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.04562970995903015,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.03986389562487602,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.03886633738875389,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.038632310926914215,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.023112818598747253,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.02144607901573181,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.021381832659244537,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.020004820078611374,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.01987089030444622,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.01329056080430746,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.014870628714561462,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.012999767437577248,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.012006370350718498,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.60.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.1972365528345108,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.17362964153289795,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.16471315920352936,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.146344855427742,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.0901959165930748,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.08123071491718292,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.10916247963905334,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.09853184968233109,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.09375027567148209,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.07749666273593903,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.07319839298725128,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.05595826357603073,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.0480220802128315,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.04423655942082405,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.043297816067934036,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.02868014946579933,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.024406813085079193,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.024211255833506584,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.022076500579714775,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.021514680236577988,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.016569670289754868,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.017277272418141365,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.015401020646095276,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.013599471189081669,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.61.self_attn.q_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.11696053296327591,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.10952581465244293,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.10609640181064606,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.09593437612056732,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.05476268008351326,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.05157216265797615,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.06348421424627304,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.058200716972351074,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.055569522082805634,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04910231754183769,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04677046090364456,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.0322904959321022,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.027832411229610443,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02626962400972843,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.02590012177824974,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.016159120947122574,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.013537810184061527,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.013371273875236511,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.012391239404678345,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.012150312773883343,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.008515258319675922,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.008362282067537308,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.008026493713259697,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.005490847397595644,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.61.self_attn.k_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.09290360659360886,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.08696427941322327,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.08406023681163788,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.07599490880966187,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.043453920632600784,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04079942777752876,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.05066452547907829,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.04652555659413338,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.04411087557673454,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.03899550810456276,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.03714115545153618,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.025744201615452766,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.02227301150560379,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.02086658403277397,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.020521780475974083,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.012911571189761162,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.01080029085278511,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.010644063353538513,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.009903996251523495,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.00968681089580059,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.006856625899672508,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.006768621504306793,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.0064138127490878105,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.004520457703620195,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.61.self_attn.v_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.24021920561790466,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.2250928431749344,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.2200704663991928,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.19913071393966675,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.11314728111028671,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.10754995048046112,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.12855742871761322,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.11700575053691864,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.11481448262929916,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.10153922438621521,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.09675423055887222,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.06566496938467026,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.05617491155862808,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.05435563251376152,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.05389559268951416,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.03316893428564072,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.027953922748565674,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02779087983071804,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.025551756843924522,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.025257430970668793,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.017391923815011978,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.016849694773554802,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.016710514202713966,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.010952234268188477,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.61.self_attn.o_proj",
"numel": 26214400,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.17529296875,
"total_bits": 57024000.0,
"err": 0.1374213695526123,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 62266880.00000001,
"err": 0.10505443811416626,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 68820480.0,
"err": 0.09364920109510422,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.72529296875,
"total_bits": 71441920.0,
"err": 0.08001791685819626,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.22529296875,
"total_bits": 84549120.0,
"err": 0.06074638292193413,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.72529296875,
"total_bits": 97656320.0,
"err": 0.04646609351038933,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 79464320.0,
"err": 0.08237633109092712,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 81927680.0,
"err": 0.06868857890367508,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.17529296875,
"total_bits": 83238400.0,
"err": 0.06508159637451172,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.52529296875,
"total_bits": 92413440.0,
"err": 0.04525262489914894,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.662646484375,
"total_bits": 96014080.0,
"err": 0.04438793659210205,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 105678720.0,
"err": 0.04152548313140869,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 108142080.0,
"err": 0.03556319698691368,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.22529296875,
"total_bits": 110763520.0,
"err": 0.03212769702076912,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.32529296875,
"total_bits": 113384960.00000001,
"err": 0.031278036534786224,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 131893120.0,
"err": 0.022719454020261765,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.22529296875,
"total_bits": 136977920.0,
"err": 0.020748458802700043,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.32529296875,
"total_bits": 139599360.0,
"err": 0.02057930827140808,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.52529296875,
"total_bits": 144842240.0,
"err": 0.018511366099119186,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.72529296875,
"total_bits": 150085120.0,
"err": 0.018078621476888657,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 158107520.0,
"err": 0.015096152201294899,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 160570880.0,
"err": 0.017375001683831215,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.2313232421875,
"total_bits": 163350400.0,
"err": 0.014233899302780628,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 212999679.99999997,
"err": 0.015715166926383972,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.61.mlp.gate_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.14687469601631165,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.13775672018527985,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.13482682406902313,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.12227106839418411,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.06911971420049667,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.0658065676689148,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.07713617384433746,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.07117113471031189,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.07011368125677109,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.06209702789783478,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.05896860361099243,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.03935093432664871,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.03410113602876663,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.033178724348545074,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.03296031802892685,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.01967007666826248,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.01718508079648018,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.017122725024819374,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.01577118970453739,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.015632031485438347,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.010412100702524185,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.01055918913334608,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.010105832479894161,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.007214249111711979,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.61.mlp.up_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1751085069444445,
"total_bits": 153951744.0,
"err": 0.14053840935230255,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.3751085069444446,
"total_bits": 168107520.0,
"err": 0.1316995769739151,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.6251085069444446,
"total_bits": 185802240.0,
"err": 0.1289067566394806,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.7251085069444443,
"total_bits": 192880128.0,
"err": 0.1169486716389656,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.2251085069444443,
"total_bits": 228269568.0,
"err": 0.06655605882406235,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.7251085069444443,
"total_bits": 263659008.0,
"err": 0.06337359547615051,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.031277126736111,
"total_bits": 214550400.0,
"err": 0.0744941309094429,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.1251085069444446,
"total_bits": 221191680.0,
"err": 0.06854667514562607,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1751085069444445,
"total_bits": 224730624.0,
"err": 0.0675341859459877,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.5251085069444446,
"total_bits": 249503232.0,
"err": 0.05984212085604668,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6625542534722224,
"total_bits": 259231488.0,
"err": 0.056951768696308136,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.031277126736111,
"total_bits": 285329280.0,
"err": 0.03840481862425804,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.125108506944445,
"total_bits": 291970560.0,
"err": 0.033307477831840515,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.225108506944444,
"total_bits": 299048448.0,
"err": 0.03242078050971031,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.325108506944445,
"total_bits": 306126336.0,
"err": 0.03220926970243454,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.031277126736111,
"total_bits": 356108160.0,
"err": 0.019363727420568466,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.225108506944444,
"total_bits": 369827328.0,
"err": 0.01756362058222294,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.325108506944445,
"total_bits": 376905216.0,
"err": 0.01749679632484913,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.525108506944444,
"total_bits": 391060992.0,
"err": 0.016295285895466805,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.725108506944444,
"total_bits": 405216768.0,
"err": 0.016168642789125443,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.031277126736111,
"total_bits": 426887040.0,
"err": 0.010900601744651794,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.125108506944445,
"total_bits": 433528320.0,
"err": 0.011821339838206768,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.231277126736111,
"total_bits": 441042816.0,
"err": 0.010621844790875912,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.125108506944445,
"total_bits": 575086080.0,
"err": 0.009222816675901413,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
},
{
"key": "model.layers.61.mlp.down_proj",
"numel": 70778880,
"options": [
{
"desc": "0.05:3b/0.95:2b 32g s4",
"bpw": 2.1715892650462965,
"total_bits": 153702656.0,
"err": 0.09226258099079132,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.25:3b/0.75:2b 32g s4",
"bpw": 2.37529296875,
"total_bits": 168120576.0,
"err": 0.08059240132570267,
"qparams": {
"group_size": 32,
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.25:4b/0.75:2b 32g s4",
"bpw": 2.62529296875,
"total_bits": 185815296.0,
"err": 0.0753101333975792,
"qparams": {
"group_size": 32,
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.4:3b/0.5:2b 32g s4",
"bpw": 2.727144820601852,
"total_bits": 193024256.0,
"err": 0.06691576540470123,
"qparams": {
"group_size": 32,
"bits": [
4,
3,
2
],
"bits_prop": [
0.1,
0.4,
0.5
],
"scale_bits": 4
}
},
{
"desc": "0.1:4b/0.9:3b 32g s4",
"bpw": 3.227144820601852,
"total_bits": 228413696.0,
"err": 0.04212964326143265,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.2:6b/0.8:3b 32g s4",
"bpw": 3.736404079861111,
"total_bits": 264458495.99999997,
"err": 0.03726546838879585,
"qparams": {
"group_size": 32,
"bits": [
6,
3
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 128g s4",
"bpw": 3.0313232421875,
"total_bits": 214553664.0,
"err": 0.053848814219236374,
"qparams": {
"group_size": 128,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:3b 32g s4",
"bpw": 3.12529296875,
"total_bits": 221204736.0,
"err": 0.04768533259630203,
"qparams": {
"group_size": 32,
"bits": [
3
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.05:4b/0.95:3b 32g s4",
"bpw": 3.1715892650462965,
"total_bits": 224481536.0,
"err": 0.04406242072582245,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"desc": "0.4:4b/0.6:3b 32g s4",
"bpw": 3.523441116898148,
"total_bits": 249385216.0,
"err": 0.03620036318898201,
"qparams": {
"group_size": 32,
"bits": [
4,
3
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.6:4b/0.4:3b 64g s4",
"bpw": 3.6644983362268517,
"total_bits": 259369088.0,
"err": 0.03451119363307953,
"qparams": {
"group_size": 64,
"bits": [
4,
3
],
"bits_prop": [
0.6,
0.4
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 128g s4",
"bpw": 4.0313232421875,
"total_bits": 285332544.0,
"err": 0.027636103332042694,
"qparams": {
"group_size": 128,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:4b 32g s4",
"bpw": 4.12529296875,
"total_bits": 291983616.0,
"err": 0.023601362481713295,
"qparams": {
"group_size": 32,
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:5b/0.9:4b 32g s4",
"bpw": 4.227144820601852,
"total_bits": 299192576.0,
"err": 0.021145569160580635,
"qparams": {
"group_size": 32,
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:4b 32g s4",
"bpw": 4.3289966724537035,
"total_bits": 306401536.0,
"err": 0.020513109862804413,
"qparams": {
"group_size": 32,
"bits": [
6,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:5b 128g s4",
"bpw": 5.0313232421875,
"total_bits": 356111424.0,
"err": 0.014322521165013313,
"qparams": {
"group_size": 128,
"bits": [
5
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:6b/0.9:5b 32g s4",
"bpw": 5.227144820601852,
"total_bits": 369971456.0,
"err": 0.012343637645244598,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.05:8b/0.05:6b/0.9:5b 32g s4",
"bpw": 5.310478153935185,
"total_bits": 375869696.0,
"err": 0.01217895932495594,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.05,
0.9
],
"scale_bits": 4
}
},
{
"desc": "0.4:6b/0.6:5b 32g s4",
"bpw": 5.523441116898148,
"total_bits": 390942976.0,
"err": 0.01134338229894638,
"qparams": {
"group_size": 32,
"bits": [
6,
5
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.3:6b/0.6:5b 32g s4",
"bpw": 5.727144820601852,
"total_bits": 405360896.0,
"err": 0.010994967073202133,
"qparams": {
"group_size": 32,
"bits": [
8,
6,
5
],
"bits_prop": [
0.1,
0.3,
0.6
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 128g s4",
"bpw": 6.0313232421875,
"total_bits": 426890304.0,
"err": 0.008539930917322636,
"qparams": {
"group_size": 128,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "1.0:6b 32g s4",
"bpw": 6.12529296875,
"total_bits": 433541376.0,
"err": 0.009497469291090965,
"qparams": {
"group_size": 32,
"bits": [
6
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
{
"desc": "0.1:8b/0.9:6b 128g s4",
"bpw": 6.235026945891204,
"total_bits": 441308224.0,
"err": 0.007764111738651991,
"qparams": {
"group_size": 128,
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"desc": "1.0:8b 32g s4",
"bpw": 8.12529296875,
"total_bits": 575099136.0,
"err": 0.007982772774994373,
"qparams": {
"group_size": 32,
"bits": [
8
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
}
]
}
],
"last_module_idx": 126,
"base_perplexity": 6.969620814022142
}