{
"measurement": {
"model.layers.0.self_attn": [
{
"accuracy": 0.9856998324394226,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9871282577514648,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936970472335815,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944248795509338,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956521987915039,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963471293449402,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961916208267212,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.996951699256897,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966274499893188,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966875314712524,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977682828903198,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975234270095825,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977285265922546,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975388646125793,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982280731201172,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981716275215149,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982712268829346,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982162714004517,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984551072120667,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.0.mlp": [
{
"accuracy": 0.9951965808868408,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953421950340271,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962496757507324,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.996468186378479,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981189966201782,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982081055641174,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986114501953125,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.999011218547821,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990652799606323,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990477561950684,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990963339805603,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994525909423828,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993903040885925,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996036291122437,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996285438537598,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996697306632996,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996899366378784,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.1.self_attn": [
{
"accuracy": 0.9951075315475464,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955105781555176,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996366560459137,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996464729309082,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976180195808411,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977752566337585,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977986216545105,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979557991027832,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985410571098328,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985666275024414,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989129304885864,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989391565322876,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989351630210876,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998952329158783,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991324543952942,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990801811218262,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991347789764404,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991260766983032,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992420673370361,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.1.mlp": [
{
"accuracy": 0.9906018376350403,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908600449562073,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929113388061523,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936069250106812,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959188103675842,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961601495742798,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971070289611816,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977706670761108,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979488253593445,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.997892439365387,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980599284172058,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.998822808265686,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987426400184631,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991464018821716,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992383718490601,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993417263031006,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993816614151001,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.2.self_attn": [
{
"accuracy": 0.9916056990623474,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927663207054138,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941953420639038,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956691265106201,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965017437934875,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966318011283875,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974705576896667,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975941777229309,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977280497550964,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977562427520752,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998244047164917,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982918500900269,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984342455863953,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998428225517273,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989588260650635,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987627863883972,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990211725234985,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988574981689453,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992058873176575,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.2.mlp": [
{
"accuracy": 0.9778180718421936,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9812322854995728,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9757185578346252,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9761385917663574,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935407042503357,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950641393661499,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956566095352173,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963124394416809,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960072636604309,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967606663703918,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.996304452419281,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972248673439026,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.997149646282196,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975483417510986,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974967837333679,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975336790084839,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975895285606384,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.3.self_attn": [
{
"accuracy": 0.9901235699653625,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901413917541504,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935774207115173,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.994044840335846,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941548705101013,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942024946212769,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944864511489868,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.994489848613739,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945494532585144,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945990443229675,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969236254692078,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979668259620667,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969637989997864,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980236887931824,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981548190116882,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988154768943787,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981858134269714,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994547963142395,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994502663612366,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.3.mlp": [
{
"accuracy": 0.9947140216827393,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947977662086487,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961393475532532,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965083599090576,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975292682647705,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977188110351562,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982392191886902,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.998745322227478,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988727569580078,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987478256225586,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989026784896851,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.999361515045166,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993957281112671,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.999619722366333,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996321201324463,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.999704897403717,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997686743736267,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.4.self_attn": [
{
"accuracy": 0.9863086342811584,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9863719344139099,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914885759353638,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919412136077881,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923102259635925,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928255081176758,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924670457839966,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929372668266296,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954795837402344,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955025911331177,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960232973098755,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972749352455139,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960730075836182,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973284602165222,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975419640541077,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987666606903076,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975559711456299,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994311332702637,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992756247520447,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.4.mlp": [
{
"accuracy": 0.9931874871253967,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933210015296936,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947327971458435,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951158761978149,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967837929725647,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970343708992004,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975770115852356,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983716011047363,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985345005989075,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983714818954468,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985796213150024,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991768598556519,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.999232292175293,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995176792144775,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995368719100952,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996165633201599,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9997265934944153,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.5.self_attn": [
{
"accuracy": 0.9904069304466248,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907057881355286,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.992792010307312,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933972358703613,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934659600257874,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942212104797363,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938386082649231,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946509599685669,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959180951118469,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959051609039307,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965561628341675,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978600740432739,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966593980789185,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979665875434875,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978505373001099,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998837947845459,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978747963905334,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994716644287109,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993830323219299,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.5.mlp": [
{
"accuracy": 0.992070198059082,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922400712966919,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937633872032166,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941806197166443,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962921738624573,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965765476226807,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971749782562256,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981174468994141,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983000755310059,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981238842010498,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.998356282711029,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990453124046326,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990988969802856,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994295239448547,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994484186172485,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995326995849609,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9996523261070251,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.6.self_attn": [
{
"accuracy": 0.9859322309494019,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9861189723014832,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900023341178894,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910764098167419,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916638731956482,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927270412445068,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922657012939453,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.993409276008606,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945438504219055,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945592880249023,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964194893836975,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972296357154846,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965633153915405,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997337818145752,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979259371757507,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985811114311218,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979987740516663,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990034103393555,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992007613182068,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.6.mlp": [
{
"accuracy": 0.990138053894043,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903435707092285,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922443628311157,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927794933319092,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952709078788757,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956415295600891,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963905215263367,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975658655166626,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978042244911194,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.997610330581665,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979149699211121,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.998786211013794,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988634586334229,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992775917053223,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993045330047607,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994122982025146,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995725154876709,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.7.self_attn": [
{
"accuracy": 0.9875317811965942,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9879106879234314,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905864596366882,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918524622917175,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921237230300903,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99309241771698,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927566647529602,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937601685523987,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.995483934879303,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955432415008545,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961994886398315,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974319338798523,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963691830635071,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976024031639099,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974542260169983,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987130165100098,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975170493125916,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992647171020508,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992377161979675,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.7.mlp": [
{
"accuracy": 0.987747311592102,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880101084709167,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903302788734436,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.990959107875824,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940916895866394,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945708513259888,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.995454728603363,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970108866691589,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973112344741821,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970077276229858,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974034428596497,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984984397888184,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986237287521362,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991340637207031,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991646409034729,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992979764938354,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9995285868644714,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.8.self_attn": [
{
"accuracy": 0.9858012795448303,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9859350919723511,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895880818367004,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905644655227661,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.990745484828949,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921642541885376,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912452697753906,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926995038986206,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952654838562012,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953035116195679,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.995283842086792,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971507787704468,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954266548156738,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972895979881287,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970881938934326,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998687744140625,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971347451210022,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994097948074341,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992066025733948,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.8.mlp": [
{
"accuracy": 0.9862116575241089,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865328073501587,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9889264106750488,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895856976509094,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933415055274963,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938807487487793,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947963953018188,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966198801994324,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969577193260193,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.996624231338501,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970693588256836,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983041882514954,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984395503997803,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990133047103882,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.999050498008728,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991880059242249,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994551539421082,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.9.self_attn": [
{
"accuracy": 0.9833400845527649,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9835159182548523,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890507459640503,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900475740432739,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.990829348564148,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.991492748260498,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915798306465149,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922983050346375,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945527911186218,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945553541183472,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950984120368958,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967748522758484,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952608346939087,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968864321708679,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969638586044312,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981995224952698,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997031033039093,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991030693054199,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991380572319031,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.9.mlp": [
{
"accuracy": 0.9848701357841492,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9852254986763,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9878257513046265,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885575771331787,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926807284355164,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932681918144226,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942807555198669,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962840676307678,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966531991958618,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963001608848572,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.996784508228302,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981482028961182,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983024001121521,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989340901374817,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989722371101379,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991272687911987,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9994289875030518,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.10.self_attn": [
{
"accuracy": 0.9815057516098022,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9818463921546936,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9868184924125671,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880124926567078,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9888033866882324,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908441305160522,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9898113012313843,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916462302207947,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933674335479736,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936434626579285,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938822388648987,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956200122833252,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942082166671753,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995800256729126,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965187907218933,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981883764266968,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996593177318573,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989185333251953,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.99880450963974,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.10.mlp": [
{
"accuracy": 0.9839500784873962,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9843326807022095,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9870963096618652,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9878791570663452,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922338128089905,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928526878356934,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939388036727905,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960058927536011,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963998198509216,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960577487945557,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965595006942749,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980032444000244,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981337189674377,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988042116165161,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988700151443481,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990306496620178,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993191957473755,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.11.self_attn": [
{
"accuracy": 0.9802574515342712,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9805864691734314,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9853855967521667,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9868611097335815,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9888980984687805,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9897518157958984,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900016188621521,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909148216247559,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.992452085018158,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.992427408695221,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943158030509949,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956783652305603,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945586919784546,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959966540336609,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966421127319336,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978750944137573,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967764616012573,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988201856613159,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989010691642761,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.11.mlp": [
{
"accuracy": 0.9821144938468933,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9825424551963806,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9855981469154358,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9864605069160461,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912657737731934,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919687509536743,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931610226631165,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955533146858215,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959957003593445,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955704212188721,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961566925048828,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977851510047913,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979726076126099,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987229108810425,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987727403640747,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989563822746277,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993228316307068,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.12.self_attn": [
{
"accuracy": 0.9803224205970764,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9806800484657288,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9854177832603455,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9872198104858398,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880571365356445,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899335503578186,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.988925039768219,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909374117851257,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939159154891968,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938865303993225,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945956468582153,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996304988861084,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948218464851379,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965044260025024,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967103004455566,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981411695480347,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968345165252686,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989164471626282,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989263415336609,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.12.mlp": [
{
"accuracy": 0.9807167053222656,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9811998009681702,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9843770265579224,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9852659702301025,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905630350112915,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913280606269836,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925594329833984,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952107667922974,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956790804862976,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952217936515808,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958574771881104,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976179599761963,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978359937667847,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986422061920166,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986936450004578,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988872408866882,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9993100762367249,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.13.self_attn": [
{
"accuracy": 0.9788755178451538,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9791787266731262,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9863776564598083,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9877495765686035,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9882678985595703,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906513094902039,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.989015519618988,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915289878845215,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945269823074341,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944683909416199,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948174953460693,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963542819023132,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995029628276825,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966146349906921,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963066577911377,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980074167251587,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963759183883667,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990120530128479,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988402128219604,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.13.mlp": [
{
"accuracy": 0.9793313145637512,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9798563718795776,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9831308722496033,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.984047532081604,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9898685216903687,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906895160675049,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919533729553223,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948509931564331,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953526258468628,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948627948760986,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955468773841858,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974362850189209,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.997664213180542,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985293745994568,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985858798027039,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987828731536865,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992362856864929,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.14.self_attn": [
{
"accuracy": 0.9735957384109497,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9742761254310608,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9820327162742615,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9838461875915527,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9850301742553711,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9875164031982422,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.986147940158844,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9887754321098328,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921441078186035,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921633005142212,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927895069122314,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950286746025085,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929989576339722,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953228831291199,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954388737678528,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974545240402222,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995554506778717,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986756443977356,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985328912734985,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.14.mlp": [
{
"accuracy": 0.9773909449577332,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9779590368270874,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9814953804016113,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9824906587600708,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.988876461982727,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9897966980934143,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911594390869141,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943316578865051,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948878288269043,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943484663963318,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.995106041431427,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.997170627117157,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974136352539062,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.998356282711029,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984269738197327,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986376166343689,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991257786750793,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.15.self_attn": [
{
"accuracy": 0.9717442989349365,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9719981551170349,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9818887114524841,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9833415746688843,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9838994145393372,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9861716628074646,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9847688674926758,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9870697855949402,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917161464691162,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917508363723755,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921398758888245,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948886036872864,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923725128173828,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.99512779712677,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.994875431060791,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974587559700012,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949555397033691,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986608028411865,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982590079307556,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.15.mlp": [
{
"accuracy": 0.9761114716529846,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9767089486122131,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9804126620292664,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9814428091049194,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9882693886756897,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892257452011108,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906419515609741,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940124154090881,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945881962776184,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940341711044312,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948209524154663,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970033764839172,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972370862960815,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982402324676514,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983088374137878,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985218644142151,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990155696868896,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.16.self_attn": [
{
"accuracy": 0.9739086627960205,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.974287211894989,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9817481637001038,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9834132790565491,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9843897223472595,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9860861897468567,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9855701923370361,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9872305393218994,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908190369606018,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912176132202148,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924086928367615,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947452545166016,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.992707371711731,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949885010719299,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952911138534546,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973462224006653,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954202771186829,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985905885696411,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987099170684814,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.16.mlp": [
{
"accuracy": 0.974743127822876,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9753699898719788,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9793030023574829,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9803747534751892,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9875796437263489,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.988615870475769,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901043772697449,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936922788619995,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943059682846069,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.993694543838501,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945386052131653,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968380928039551,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971153140068054,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981586933135986,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982346296310425,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984610676765442,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990023374557495,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.17.self_attn": [
{
"accuracy": 0.9733713865280151,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9737836122512817,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9813607931137085,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9833952784538269,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9849681258201599,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9863169193267822,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9863423705101013,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9878222942352295,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911315441131592,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908773899078369,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924609065055847,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948031306266785,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927574992179871,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995141327381134,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955845475196838,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997370719909668,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957453608512878,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984058141708374,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987477660179138,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.17.mlp": [
{
"accuracy": 0.9727590680122375,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9734351634979248,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9776291251182556,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9787915349006653,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865829348564148,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9877066016197205,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892941117286682,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931706190109253,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938337802886963,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931781888008118,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940930604934692,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965726137161255,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968577027320862,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979869723320007,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980683326721191,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983075857162476,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988787174224854,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.18.self_attn": [
{
"accuracy": 0.9716711044311523,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9719977974891663,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9782347679138184,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9799559712409973,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9840918183326721,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9856959581375122,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9854209423065186,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9870882630348206,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912691116333008,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915600419044495,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918767809867859,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941932559013367,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921875596046448,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945084452629089,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950032830238342,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970414042472839,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951456785202026,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981731176376343,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983041286468506,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.18.mlp": [
{
"accuracy": 0.9703275561332703,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.971074104309082,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9755733013153076,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9768006801605225,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9854116439819336,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9866145849227905,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9883067011833191,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925618171691895,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932701587677002,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925706386566162,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935535192489624,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962499737739563,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965547323226929,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977787137031555,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978973865509033,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981514811515808,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987823367118835,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.19.self_attn": [
{
"accuracy": 0.970288872718811,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9700698256492615,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9814435243606567,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9831294417381287,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.98454749584198,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9866775274276733,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9857820272445679,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880163073539734,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920047521591187,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919849038124084,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928273558616638,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948750138282776,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931305050849915,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950854182243347,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958880543708801,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974316954612732,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996042788028717,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984602332115173,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987090229988098,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.19.mlp": [
{
"accuracy": 0.9677940607070923,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9686390161514282,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9735495448112488,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9748945832252502,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9842025637626648,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9855004549026489,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9873441457748413,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919409155845642,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.992695152759552,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919539093971252,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930076003074646,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959323406219482,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962307214736938,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975703954696655,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976417422294617,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979098439216614,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985460042953491,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.20.self_attn": [
{
"accuracy": 0.9782660603523254,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9787476658821106,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9835131764411926,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9858707785606384,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9889205098152161,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9894518256187439,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915178418159485,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920823574066162,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935387969017029,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936754703521729,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947386980056763,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995438814163208,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953460693359375,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960796236991882,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997427761554718,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978749752044678,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997780978679657,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985519647598267,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991057515144348,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.20.mlp": [
{
"accuracy": 0.9645863771438599,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9668954014778137,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9698701500892639,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9708445072174072,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9855899810791016,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9863011837005615,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9879109263420105,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920929074287415,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.992228090763092,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925156831741333,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927776455879211,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959104061126709,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958609938621521,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970345497131348,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976388216018677,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978735446929932,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998342752456665,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.21.self_attn": [
{
"accuracy": 0.9859275221824646,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9864243865013123,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9883950352668762,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907479286193848,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930896759033203,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932896494865417,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954572319984436,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957019686698914,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960008859634399,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964379668235779,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965173006057739,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967730045318604,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969896674156189,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973520636558533,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983364343643188,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984691143035889,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986892938613892,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988846778869629,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999336838722229,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.21.mlp": [
{
"accuracy": 0.9727259874343872,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.973426342010498,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9776137471199036,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9787774085998535,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865185618400574,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9876039028167725,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.989178478717804,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929096698760986,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935342669487,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931319952011108,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.994020938873291,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965089559555054,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967576861381531,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978557229042053,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980059862136841,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982372522354126,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987640976905823,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.22.self_attn": [
{
"accuracy": 0.9825988411903381,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9828034043312073,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9861343502998352,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890449047088623,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910181164741516,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913138747215271,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943740963935852,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946104288101196,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948292970657349,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952729940414429,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955911040306091,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960383772850037,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963269829750061,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966481328010559,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977351427078247,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979622960090637,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981664419174194,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983905553817749,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988709688186646,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.22.mlp": [
{
"accuracy": 0.9411613941192627,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9413696527481079,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9428952932357788,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9433971643447876,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9764400124549866,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9804890155792236,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.981490433216095,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9813287258148193,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9836829900741577,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.984821081161499,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9852535724639893,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9856575131416321,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9856362342834473,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.985938310623169,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904977083206177,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905548691749573,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906831979751587,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.23.self_attn": [
{
"accuracy": 0.9816914796829224,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9823886156082153,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9850608110427856,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9888694286346436,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.990800142288208,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909163117408752,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942601323127747,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943732619285583,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952013492584229,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953919649124146,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952986240386963,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957552552223206,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960126280784607,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964054226875305,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978461861610413,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980822205543518,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983677864074707,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986541867256165,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991601705551147,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.23.mlp": [
{
"accuracy": 0.9722568392753601,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.972943127155304,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9773343205451965,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9785789847373962,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9862989783287048,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9874355792999268,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9891331791877747,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930184483528137,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.993684709072113,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930380582809448,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939634203910828,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964966773986816,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968082904815674,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979424476623535,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980531930923462,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983132481575012,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989029169082642,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.24.self_attn": [
{
"accuracy": 0.9802296757698059,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9808048605918884,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9831228852272034,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9871618151664734,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9897360801696777,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899426698684692,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935844540596008,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938173890113831,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945397973060608,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947910308837891,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950952529907227,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954748749732971,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958120584487915,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961535334587097,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976428747177124,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978353977203369,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982757568359375,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984474182128906,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990204572677612,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.24.mlp": [
{
"accuracy": 0.9699947834014893,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9707400798797607,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9755772948265076,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9769413471221924,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.985196053981781,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9864394068717957,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9883006811141968,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924659132957458,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932010173797607,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924830198287964,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.993498682975769,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962387084960938,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965773224830627,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978121519088745,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.997924268245697,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982140064239502,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988653659820557,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.25.self_attn": [
{
"accuracy": 0.9777749180793762,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9784379005432129,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.981274425983429,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9858444929122925,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9882760047912598,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884033203125,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926275610923767,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928050637245178,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938842058181763,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941917657852173,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944493174552917,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.994939923286438,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953437447547913,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957882761955261,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973412156105042,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997627317905426,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.998013973236084,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982996582984924,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989508986473083,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.25.mlp": [
{
"accuracy": 0.9678333401679993,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9686379432678223,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9738751649856567,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9753829836845398,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9841237664222717,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9854159355163574,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9874625205993652,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918787479400635,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.992668092250824,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919316172599792,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.992996335029602,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959583878517151,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962957501411438,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976390600204468,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977637529373169,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980834722518921,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987630844116211,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.26.self_attn": [
{
"accuracy": 0.9749915599822998,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.975801408290863,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9792172312736511,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9835798740386963,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9873948693275452,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9877035617828369,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918880462646484,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.992158055305481,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929216504096985,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9932470321655273,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.993950605392456,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944664239883423,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948474168777466,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995285153388977,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970676898956299,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973377585411072,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977148771286011,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980754852294922,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988669157028198,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.26.mlp": [
{
"accuracy": 0.9671778082847595,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.967981219291687,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9733801484107971,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9749508500099182,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9837911128997803,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9850966930389404,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9872438311576843,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.991705060005188,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925094842910767,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917752146720886,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928532838821411,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958907961845398,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962350726127625,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976212382316589,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977555871009827,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980970025062561,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998808741569519,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.27.self_attn": [
{
"accuracy": 0.9713654518127441,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9721508026123047,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9756113886833191,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9798527956008911,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9851890206336975,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9854009747505188,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.989967405796051,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901362061500549,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904399514198303,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907974600791931,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.992651641368866,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931778907775879,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.993507444858551,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939098954200745,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965385794639587,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968758821487427,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973697662353516,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979129433631897,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987874031066895,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.27.mlp": [
{
"accuracy": 0.9657012224197388,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9665738344192505,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9721505641937256,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9737791419029236,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.983054518699646,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9844212532043457,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9866183400154114,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913182854652405,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921499490737915,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913854598999023,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925199151039124,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956878423690796,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960508942604065,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974879026412964,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976134896278381,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979585409164429,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986864328384399,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.28.self_attn": [
{
"accuracy": 0.9707354307174683,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.971511721611023,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9752561450004578,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9797199964523315,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9850357174873352,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9852256774902344,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.990052342414856,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903844594955444,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915469884872437,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918392896652222,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927065968513489,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933715462684631,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.993642270565033,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942557215690613,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963071346282959,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967182278633118,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970980286598206,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997718870639801,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986751675605774,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.28.mlp": [
{
"accuracy": 0.9637611508369446,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9646869897842407,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9704722166061401,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9721781015396118,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9820609092712402,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9835200905799866,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9858158230781555,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908035397529602,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916855096817017,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9908860325813293,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920897483825684,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954417943954468,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958270192146301,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973486065864563,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974896311759949,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978528618812561,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986364841461182,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.29.self_attn": [
{
"accuracy": 0.9666864275932312,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9672436714172363,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9716740846633911,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9763109683990479,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9837266206741333,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9840741157531738,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885584115982056,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9888198375701904,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.990667462348938,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910120964050293,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920483827590942,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927121996879578,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930009245872498,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.993488609790802,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.996083676815033,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964937567710876,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969232082366943,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975435137748718,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985467195510864,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.29.mlp": [
{
"accuracy": 0.9620561003684998,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9630187153816223,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.96901535987854,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9708089828491211,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9811897277832031,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9827054738998413,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9851159453392029,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903323650360107,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912667870521545,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904359579086304,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916917085647583,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952090978622437,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.995604395866394,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971993565559387,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973751306533813,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977596402168274,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985827207565308,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.30.self_attn": [
{
"accuracy": 0.9711217880249023,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9718441963195801,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.975135326385498,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9804184436798096,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9853742122650146,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9857009053230286,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.990673840045929,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910926818847656,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918697476387024,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922826886177063,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925054311752319,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931897521018982,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934414625167847,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941400289535522,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963738918304443,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968234896659851,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972355961799622,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977129697799683,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986679553985596,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.30.mlp": [
{
"accuracy": 0.9605146646499634,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9615113139152527,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9678134322166443,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9696913957595825,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9803912043571472,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9819815754890442,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9845199584960938,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.989915132522583,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909040331840515,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900276064872742,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913426041603088,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950181245803833,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954156279563904,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970999360084534,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972459673881531,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976488947868347,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984903335571289,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.31.self_attn": [
{
"accuracy": 0.96865314245224,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9694852232933044,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9732229709625244,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9787383675575256,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9843489527702332,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9846925139427185,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899484515190125,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903217554092407,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916084408760071,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913873076438904,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920704960823059,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927394390106201,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930543303489685,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.993716299533844,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961820244789124,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966893196105957,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971513152122498,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977155923843384,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987204670906067,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.31.mlp": [
{
"accuracy": 0.959182620048523,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9601984024047852,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9667444229125977,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9687039256095886,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9796993732452393,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9813438057899475,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9840139150619507,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895626902580261,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905890822410583,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896793961524963,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910441040992737,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948462247848511,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.995270311832428,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970093369483948,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971627593040466,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975890517234802,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984620809555054,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.32.self_attn": [
{
"accuracy": 0.964709460735321,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9655798077583313,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9690863490104675,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9749228954315186,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9818655848503113,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9822431802749634,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9881715774536133,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9886343479156494,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9898155331611633,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905335903167725,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910184741020203,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918739795684814,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920876622200012,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.992939829826355,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956647753715515,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961290955543518,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967518448829651,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973429441452026,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984892010688782,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.32.mlp": [
{
"accuracy": 0.9569997191429138,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9580748081207275,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9649038910865784,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9669493436813354,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9786027073860168,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9803602695465088,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9831362962722778,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890173673629761,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901048541069031,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9891321063041687,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905844330787659,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945818781852722,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950490593910217,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968757629394531,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970235228538513,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974698424339294,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984118342399597,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.33.self_attn": [
{
"accuracy": 0.9635481834411621,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.964462399482727,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.968926727771759,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9750816822052002,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.981317400932312,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9816939234733582,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.987581193447113,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880806803703308,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9893903136253357,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9898027777671814,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.990487277507782,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915239810943604,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916298389434814,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.992660403251648,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952960014343262,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958807229995728,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962772727012634,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997062087059021,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998467743396759,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.33.mlp": [
{
"accuracy": 0.9543027281761169,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9554525017738342,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9626349806785583,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.964832603931427,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9771948456764221,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9790633320808411,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9820157289505005,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.988258957862854,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9894247055053711,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884110689163208,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899502992630005,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942091703414917,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947029948234558,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966431260108948,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968225955963135,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972978234291077,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982929229736328,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.34.self_attn": [
{
"accuracy": 0.961439847946167,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9623470902442932,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9674926400184631,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9731425046920776,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9803644418716431,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9808581471443176,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9866241216659546,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9870777130126953,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9887080788612366,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9891021251678467,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9902105331420898,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913060069084167,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.991320013999939,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925051331520081,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950475096702576,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958486557006836,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959821701049805,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970235824584961,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980623126029968,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.34.mlp": [
{
"accuracy": 0.9514939785003662,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9526727199554443,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9602830410003662,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9626453518867493,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9757739305496216,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9777461290359497,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9809086918830872,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9874991774559021,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.988744854927063,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9876909255981445,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9893155097961426,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.993840754032135,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943646192550659,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964222311973572,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966251254081726,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971365928649902,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998183012008667,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.35.self_attn": [
{
"accuracy": 0.9592726826667786,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9604480266571045,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9650497436523438,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9715477824211121,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9792090654373169,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9797958135604858,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.985846221446991,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865691065788269,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9878040552139282,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9882896542549133,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892797470092773,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903238415718079,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904813766479492,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915562868118286,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947348237037659,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955733418464661,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995773434638977,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968797564506531,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979491829872131,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.35.mlp": [
{
"accuracy": 0.9479681849479675,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9492290616035461,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9573999643325806,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9600135087966919,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9740007519721985,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9761069416999817,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.979552149772644,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865307211875916,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9878835082054138,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9867833852767944,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885157346725464,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933667182922363,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939089417457581,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961151480674744,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963293671607971,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9968801140785217,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979541897773743,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.36.self_attn": [
{
"accuracy": 0.9516373872756958,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9526516795158386,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.96016925573349,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9659306406974792,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9748197793960571,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9758643507957458,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9808191061019897,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9820675849914551,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9838852286338806,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9836902022361755,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9875097274780273,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9889787435531616,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885905385017395,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900829792022705,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937109351158142,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945994019508362,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947569370269775,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966885447502136,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980640411376953,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.36.mlp": [
{
"accuracy": 0.9439328908920288,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9452832341194153,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9539304971694946,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9566988945007324,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9719969034194946,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9742728471755981,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.977906346321106,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9855319857597351,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9869711995124817,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.985792875289917,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9876554608345032,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928741455078125,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935021996498108,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958592653274536,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961082935333252,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966993927955627,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979169964790344,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.37.self_attn": [
{
"accuracy": 0.955241322517395,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9565101265907288,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9621816277503967,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9696477651596069,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9772909879684448,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9776322841644287,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9846312403678894,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9849592447280884,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9868574738502502,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9873914122581482,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9879876971244812,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9890944957733154,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.989226758480072,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904196858406067,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936864376068115,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946553707122803,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947550296783447,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962059259414673,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975231885910034,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.37.mlp": [
{
"accuracy": 0.9386700987815857,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9401164650917053,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9497803449630737,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9529090523719788,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9693139791488647,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9718050956726074,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9759001135826111,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9840950965881348,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9857105016708374,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9844117760658264,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9864606857299805,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921727776527405,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928423762321472,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954302310943604,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956881999969482,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963475465774536,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976211786270142,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.38.self_attn": [
{
"accuracy": 0.9527000188827515,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9537625908851624,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9631722569465637,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9700884819030762,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9769263863563538,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9773066639900208,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9835425019264221,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9839573502540588,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9866557121276855,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9870819449424744,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.988139808177948,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896345138549805,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892482161521912,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907289147377014,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940725564956665,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951866865158081,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951623678207397,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969174861907959,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981357455253601,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.38.mlp": [
{
"accuracy": 0.9368841648101807,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9383493065834045,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9482523798942566,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9514368176460266,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.968390166759491,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9709649085998535,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9751624464988708,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9836687445640564,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9853178262710571,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9839651584625244,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.986070454120636,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9919387698173523,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926680326461792,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953103065490723,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955769777297974,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962509870529175,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975844621658325,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.39.self_attn": [
{
"accuracy": 0.9560840129852295,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9575971961021423,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9644007682800293,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9704262018203735,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9765982627868652,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9774425625801086,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9822746515274048,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9832584261894226,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9854255318641663,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865472316741943,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.988345205783844,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9898467659950256,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9892755746841431,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.990959107875824,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943384528160095,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954100251197815,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952843189239502,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969438910484314,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981462359428406,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.39.mlp": [
{
"accuracy": 0.9339728951454163,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9354652166366577,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.946017861366272,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.949378252029419,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.966935396194458,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.969651460647583,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9741052389144897,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9829458594322205,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9846770763397217,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9832428097724915,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9854462146759033,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915997385978699,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.992372453212738,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951454401016235,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.995441198348999,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961680769920349,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975989460945129,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.40.self_attn": [
{
"accuracy": 0.957836925983429,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9593684673309326,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9652334451675415,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9710041284561157,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9788455963134766,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9794567823410034,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9844955205917358,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9852043986320496,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9869250059127808,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9870510101318359,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899078607559204,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909328818321228,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909846782684326,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917073249816895,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.994604766368866,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955381751060486,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955064654350281,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971497058868408,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9982522130012512,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.40.mlp": [
{
"accuracy": 0.9315168857574463,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9330423474311829,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9439978003501892,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9474522471427917,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9656731486320496,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.968481183052063,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9731139540672302,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9823104739189148,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9841135740280151,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9825737476348877,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9848874807357788,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912592768669128,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.992084264755249,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949619770050049,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952108860015869,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959546327590942,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997422993183136,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.41.self_attn": [
{
"accuracy": 0.9551436901092529,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9563466310501099,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9627535939216614,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9694514870643616,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9762164950370789,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9773015379905701,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9827701449394226,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9840812087059021,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9858297109603882,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.986511766910553,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9883401989936829,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895725250244141,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895883202552795,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909176230430603,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938281178474426,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947828650474548,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948920607566833,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963607788085938,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998085081577301,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.41.mlp": [
{
"accuracy": 0.9303711652755737,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9319008588790894,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9429486393928528,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9464423656463623,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9650914072990417,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9679352641105652,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9725956320762634,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9819976091384888,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9838343262672424,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9822766780853271,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9846240282058716,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9911145567893982,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.991949737071991,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9948809742927551,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951489567756653,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959022998809814,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974055290222168,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.42.self_attn": [
{
"accuracy": 0.9596095681190491,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9606311917304993,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9692485332489014,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9741523265838623,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9802128076553345,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9811873435974121,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9849035739898682,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9861278533935547,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9881898760795593,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885151982307434,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901829361915588,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913119077682495,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910102486610413,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921627640724182,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946861267089844,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956772923469543,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954618215560913,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972682595252991,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985305666923523,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.42.mlp": [
{
"accuracy": 0.9314454197883606,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.932918906211853,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9436354637145996,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9469728469848633,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9656684994697571,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9684316515922546,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9729456901550293,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9823508858680725,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9841389656066895,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9826032519340515,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.984889030456543,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913023114204407,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921433329582214,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950590133666992,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953010082244873,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960380792617798,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975926876068115,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.43.self_attn": [
{
"accuracy": 0.9553138017654419,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9560855627059937,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9652404189109802,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9699717164039612,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9794797301292419,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9803782105445862,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9843900799751282,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9855085611343384,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9873715043067932,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880423545837402,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895517826080322,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907999038696289,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903549551963806,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9917619228363037,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945350885391235,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954448938369751,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953585267066956,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970747828483582,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979410171508789,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.43.mlp": [
{
"accuracy": 0.9319198131561279,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9334291219711304,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9437958002090454,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9469906687736511,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9659423232078552,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9686774015426636,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9730249047279358,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9825268983840942,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.984283983707428,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9827377200126648,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9850094318389893,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.991375744342804,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.992216944694519,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951170086860657,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953281283378601,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960341453552246,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976003170013428,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.44.self_attn": [
{
"accuracy": 0.960800051689148,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9618864059448242,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9680581092834473,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9734706282615662,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9801445603370667,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9812307953834534,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9857293963432312,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9871036410331726,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9886781573295593,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.989132285118103,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9903171062469482,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9910436868667603,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912834763526917,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920168519020081,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951731562614441,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958404898643494,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961391687393188,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972615838050842,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983828663825989,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.44.mlp": [
{
"accuracy": 0.9334535598754883,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9349251985549927,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9447961449623108,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9478158950805664,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9667091965675354,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9693937301635742,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9735316634178162,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.982965886592865,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9846624732017517,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9831480979919434,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9853622317314148,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915875196456909,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924306869506836,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952679872512817,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9954456090927124,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961161017417908,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976840615272522,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.45.self_attn": [
{
"accuracy": 0.9672577977180481,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9677121639251709,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9727157950401306,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9766370058059692,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9823560118675232,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9836591482162476,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9866418838500977,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9881843328475952,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9895275831222534,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899613261222839,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913470149040222,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923160076141357,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920384287834167,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930799603462219,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953200817108154,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961152672767639,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960209131240845,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973070025444031,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998319685459137,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.45.mlp": [
{
"accuracy": 0.9357622265815735,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9371930360794067,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9466590285301208,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9495121836662292,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9678441286087036,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9704773426055908,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.974406898021698,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9835559129714966,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9852184057235718,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9837061762809753,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9858675599098206,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9918447136878967,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926788210868835,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953672885894775,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955559968948364,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961830973625183,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976717233657837,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.46.self_attn": [
{
"accuracy": 0.9638057947158813,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9651666283607483,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9713346362113953,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9757682085037231,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9817551374435425,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9831270575523376,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9863356947898865,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880350828170776,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9899063110351562,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9900275468826294,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.991357147693634,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922109842300415,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923080801963806,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931508302688599,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957128763198853,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963703155517578,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965291619300842,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997677206993103,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986215829849243,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.46.mlp": [
{
"accuracy": 0.9367968440055847,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9382206201553345,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9474208354949951,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.950163722038269,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9684066772460938,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9709464907646179,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9747655987739563,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9838753938674927,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9854753613471985,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9840012192726135,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9861109852790833,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920213222503662,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928411841392517,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955418705940247,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957272410392761,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963528513908386,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979128241539001,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.47.self_attn": [
{
"accuracy": 0.9685832262039185,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9691165089607239,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9745590090751648,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9781190156936646,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.98417729139328,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9848834276199341,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9883137345314026,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9891355633735657,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909599423408508,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912471175193787,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.991927981376648,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929337501525879,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927876591682434,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937881827354431,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959303736686707,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966718554496765,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965633749961853,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.997857391834259,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986227750778198,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.47.mlp": [
{
"accuracy": 0.9382524490356445,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9396401643753052,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9486047029495239,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9512426853179932,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9691209197044373,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9716330170631409,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.975318431854248,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9842603206634521,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.985832691192627,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.98436039686203,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9864382743835449,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921943545341492,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929957389831543,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956135153770447,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958470463752747,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964519739151001,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980047941207886,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.48.self_attn": [
{
"accuracy": 0.9694262742996216,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9701332449913025,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9760941863059998,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9792918562889099,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9847738742828369,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9854689836502075,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885985851287842,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9894176125526428,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912323951721191,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915620684623718,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928472638130188,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936367869377136,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935915470123291,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944106340408325,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963013529777527,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969415068626404,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969139695167542,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981604218482971,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987903833389282,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.48.mlp": [
{
"accuracy": 0.9385270476341248,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9399098753929138,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9487301707267761,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9513040781021118,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9693220257759094,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9718074202537537,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.975415825843811,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9844034314155579,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9859479069709778,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.984490156173706,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865396618843079,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922913908958435,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930897355079651,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957319498062134,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958862662315369,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964790940284729,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980481266975403,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.49.self_attn": [
{
"accuracy": 0.9677457213401794,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9691992998123169,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9748064279556274,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9786579608917236,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9842808842658997,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9848085045814514,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.988530158996582,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.989181637763977,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9902215003967285,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906070232391357,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9922890663146973,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.992999255657196,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931091070175171,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939003586769104,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960586428642273,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967271685600281,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967123866081238,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977551698684692,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985455274581909,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.49.mlp": [
{
"accuracy": 0.9396533966064453,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9410251379013062,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9494670629501343,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9519233703613281,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9698349237442017,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9722951650619507,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9757516384124756,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9846510291099548,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9861730337142944,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9847337603569031,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9867590665817261,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9923920035362244,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931836128234863,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957578182220459,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959295988082886,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964945912361145,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980298280715942,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.50.self_attn": [
{
"accuracy": 0.9740390777587891,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9747003316879272,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9788010716438293,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9819967746734619,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865987300872803,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.987201452255249,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.990204930305481,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9909303188323975,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921326637268066,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924771785736084,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935494661331177,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941951632499695,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9942806363105774,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949588775634766,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967302083969116,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972891807556152,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973254203796387,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983274936676025,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990348219871521,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.50.mlp": [
{
"accuracy": 0.9406588077545166,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9420133233070374,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9502657651901245,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9526530504226685,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9703471660614014,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9727683663368225,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9761211276054382,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9849207997322083,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9864187240600586,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9849934577941895,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.98698490858078,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925204515457153,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933006763458252,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958251714706421,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960201978683472,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965700507164001,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981065392494202,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.51.self_attn": [
{
"accuracy": 0.9709547758102417,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.971830427646637,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.976800799369812,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9800977110862732,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9840500950813293,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9849366545677185,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9887124300003052,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9896965622901917,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915709495544434,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916844367980957,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926708936691284,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934515357017517,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938026666641235,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944905638694763,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965225458145142,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971402883529663,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9972074031829834,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981832504272461,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9988847374916077,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.51.mlp": [
{
"accuracy": 0.9417832493782043,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9431332945823669,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.951073408126831,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9533714056015015,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.970926821231842,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9732903242111206,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9765179753303528,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9852054119110107,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9866737723350525,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9852785468101501,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9872331619262695,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926603436470032,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934173226356506,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958957433700562,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960467219352722,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965683221817017,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9980412125587463,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.52.self_attn": [
{
"accuracy": 0.9771499037742615,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9778532385826111,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9816587567329407,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9839985370635986,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.988129734992981,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9886678457260132,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9914753437042236,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9920191764831543,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931519031524658,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933420419692993,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943209290504456,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995009183883667,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950110912322998,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956792593002319,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971641302108765,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976961612701416,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976643323898315,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985297918319702,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991656541824341,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.52.mlp": [
{
"accuracy": 0.9432758688926697,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.944612979888916,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9521853923797607,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.95437091588974,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9716439843177795,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9739655256271362,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9770265817642212,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9855494499206543,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9869892597198486,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.985627293586731,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9875398874282837,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928115010261536,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9935489892959595,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959484934806824,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961544871330261,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966502785682678,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998108983039856,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.53.self_attn": [
{
"accuracy": 0.9755910038948059,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9763771891593933,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9806103110313416,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9828593730926514,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9873501658439636,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880325198173523,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9907620549201965,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9916507005691528,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9926304221153259,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9928610920906067,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940903186798096,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947069883346558,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9949231743812561,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955252408981323,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971699118614197,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976937770843506,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9976733922958374,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9986196160316467,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999132513999939,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.53.mlp": [
{
"accuracy": 0.9448935985565186,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.94619220495224,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9534260034561157,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9555139541625977,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9724273085594177,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9746795296669006,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9776148200035095,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9859508275985718,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9873506426811218,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9860252737998962,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.987890362739563,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930332899093628,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9937467575073242,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960974454879761,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962440729141235,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.996717631816864,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981284737586975,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.54.self_attn": [
{
"accuracy": 0.9782835841178894,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9788364768028259,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9829731583595276,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9851095676422119,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885916113853455,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9894437789916992,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921833872795105,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929976463317871,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938375353813171,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940028786659241,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947134256362915,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952039122581482,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956178665161133,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960771203041077,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974365830421448,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.997896671295166,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979169964790344,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9987118244171143,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9992337226867676,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.54.mlp": [
{
"accuracy": 0.9458270072937012,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9470985531806946,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9541079998016357,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9561334848403931,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9729111194610596,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9751281142234802,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9779425263404846,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9861955046653748,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.987575113773346,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9862715005874634,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9881045818328857,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931491017341614,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9938469529151917,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961421489715576,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963225722312927,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9967789649963379,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981856346130371,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.55.self_attn": [
{
"accuracy": 0.9767827987670898,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9776670336723328,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9818980097770691,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9848973155021667,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9881643056869507,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9885873794555664,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924196004867554,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9929406642913818,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936953186988831,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939186573028564,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9941592812538147,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9946964383125305,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995204746723175,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.995742678642273,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9973733425140381,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977534413337708,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979586005210876,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.998581051826477,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9991828203201294,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.55.mlp": [
{
"accuracy": 0.9459520578384399,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9472599625587463,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9540058970451355,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9559538960456848,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9729585647583008,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9751941561698914,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.977882444858551,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.986138641834259,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9875190854072571,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9862169027328491,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880638122558594,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930427074432373,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936884641647339,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9959146976470947,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9962020516395569,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966230392456055,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979254007339478,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.56.self_attn": [
{
"accuracy": 0.9704675078392029,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9715405702590942,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9760439991950989,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9801476001739502,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9851670861244202,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9857860803604126,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905674457550049,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9913652539253235,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9921212196350098,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924564957618713,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927710890769958,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9933167695999146,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9940667748451233,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9945276379585266,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9966800808906555,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9971042275428772,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974862337112427,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979977011680603,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989217519760132,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.56.mlp": [
{
"accuracy": 0.9476583003997803,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.948943555355072,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9552018046379089,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.957016110420227,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9736846089363098,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9758626222610474,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9783369302749634,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9863307476043701,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9876561164855957,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9864436984062195,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9882057309150696,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9930835962295532,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9934483766555786,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957367777824402,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960263967514038,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963878989219666,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9974877238273621,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.57.self_attn": [
{
"accuracy": 0.9803853631019592,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9811383485794067,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9841658473014832,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9870100021362305,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9901928305625916,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9905272126197815,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9939978718757629,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943659901618958,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950476288795471,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952346086502075,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951736330986023,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9955679178237915,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960494637489319,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963723421096802,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9977887868881226,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979199171066284,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983218908309937,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985413551330566,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.999061644077301,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.57.mlp": [
{
"accuracy": 0.9485303163528442,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9498096108436584,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9558095335960388,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9575709700584412,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9742081761360168,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9763991832733154,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9787432551383972,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9865858554840088,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9878907799720764,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.986755907535553,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9884940981864929,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931547045707703,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9936485290527344,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9957143664360046,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961339235305786,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964829683303833,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9975903034210205,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.58.self_attn": [
{
"accuracy": 0.980771005153656,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9812217354774475,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9842353463172913,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9871218800544739,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9904466271400452,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9906569719314575,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943327307701111,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9944990873336792,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9951406717300415,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952616095542908,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9953405857086182,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956916570663452,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9961485862731934,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9963927865028381,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9978238940238953,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9979299902915955,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9983550906181335,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984583854675293,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9989752173423767,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.58.mlp": [
{
"accuracy": 0.9491286873817444,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9505853056907654,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9559405446052551,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9575273394584656,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9744371771812439,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9764898419380188,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9785716533660889,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9862229824066162,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9872124195098877,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.986594021320343,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9880166053771973,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925879240036011,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9925935864448547,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9947167634963989,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956130385398865,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9958862066268921,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.996752917766571,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.59.self_attn": [
{
"accuracy": 0.9847760796546936,
"total_bits": 248534016,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9853666424751282,
"total_bits": 255874048,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9875748157501221,
"total_bits": 261933440,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9898779988288879,
"total_bits": 311705344,
"q_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9924492239952087,
"total_bits": 368585216,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9927709698677063,
"total_bits": 368817280,
"q_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64,
"3": 64
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9952589273452759,
"total_bits": 473442816,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956172704696655,
"total_bits": 473674880,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9956095814704895,
"total_bits": 477587712,
"q_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 64
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9960986971855164,
"total_bits": 484485120,
"q_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9964334964752197,
"total_bits": 486257792,
"q_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9965994954109192,
"total_bits": 490137856,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9969613552093506,
"total_bits": 496260096,
"q_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 64
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 64,
"4": 64
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9970330595970154,
"total_bits": 503130112,
"q_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"5": 32
},
"bits": [
5
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981682300567627,
"total_bits": 610019840,
"q_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9981345534324646,
"total_bits": 620570624,
"q_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
}
},
{
"accuracy": 0.9985522031784058,
"total_bits": 708323840,
"q_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9984948039054871,
"total_bits": 734046208,
"q_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 32
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"6": 32
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9990249872207642,
"total_bits": 943204864,
"q_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.layers.59.mlp": [
{
"accuracy": 0.9461793303489685,
"total_bits": 983263488,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9482079744338989,
"total_bits": 1019963648,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"6": 32,
"3": 64,
"2": 64
},
"bits": [
6,
3,
2
],
"bits_prop": [
0.05,
0.2,
0.75
],
"scale_bits": 4
}
},
{
"accuracy": 0.9523082971572876,
"total_bits": 1139319296,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"3": 32
},
"bits": [
5,
3
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9536916613578796,
"total_bits": 1278779904,
"gate_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"3": 64,
"2": 64
},
"bits": [
3,
2
],
"bits_prop": [
0.3,
0.7
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.973205029964447,
"total_bits": 1439479680,
"gate_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"3": 128
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128,
"3": 128
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9753316640853882,
"total_bits": 1479461888,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32,
"3": 32
},
"bits": [
8,
4,
3
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9770070314407349,
"total_bits": 1591146112,
"gate_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32,
"3": 32
},
"bits": [
4,
3
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9852818250656128,
"total_bits": 1819203328,
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 128
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9861727356910706,
"total_bits": 1846070272,
"gate_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 32
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"4": 32
},
"bits": [
8,
4
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.9855901598930359,
"total_bits": 1872541568,
"gate_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 128,
"4": 128
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 128,
"4": 128
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9871061444282532,
"total_bits": 1912523776,
"gate_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"5": 32,
"4": 32
},
"bits": [
5,
4
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"5": 32,
"4": 32
},
"bits": [
8,
5,
4
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9912294745445251,
"total_bits": 2305603456,
"gate_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128,
"5": 128
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128,
"5": 128
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9915534257888794,
"total_bits": 2345585664,
"gate_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 32,
"5": 32
},
"bits": [
6,
5
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 32,
"5": 32
},
"bits": [
8,
6,
5
],
"bits_prop": [
0.05,
0.1,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9931061267852783,
"total_bits": 2671556480,
"gate_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"6": 128
},
"bits": [
6
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 32,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
}
},
{
"accuracy": 0.994154155254364,
"total_bits": 2763141888,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.15,
0.85
],
"scale_bits": 4
}
},
{
"accuracy": 0.9943525791168213,
"total_bits": 3012702976,
"gate_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128,
"6": 128
},
"bits": [
8,
6
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
{
"accuracy": 0.9950693249702454,
"total_bits": 3536990976,
"gate_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"8": 128
},
"bits": [
8
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
],
"model.norm.norm": null,
"lm_head.linear": null
},
"last_module_idx": 122
}